poyee-data-warehouse/dw_base/scheduler/ent_interface_dingtalk_call.py

import base64
import hashlib
import hmac
import sys
import re
import os
import urllib
import time
import requests

abspath = os.path.abspath(__file__)
root_path = re.sub(r"tendata-warehouse.*", "tendata-warehouse", abspath)
sys.path.append(root_path)
from dw_base.utils.config_utils import parse_args
from dw_base.spark.spark_sql import SparkSQL
import json

spark = SparkSQL(udf_files=['dw_base/spark/udf/contacts/ctc_common.py',
                            'dw_base/spark/udf/spark_id_generate_udf.py'])



def send_dingtalk_notification(msg):
    headers = {"Content-Type": "application/json"}
    data = {
        "msgtype": "text",
        "text": {"content": msg}
    }
    json_data = json.dumps(data)
    url = 'https://oapi.dingtalk.com/robot/send?access_token=5183dfe1ecbe06261bcac7b45c1a6b5ae101fec67877d74120a6a95c88d1f917'
    # url = 'https://oapi.dingtalk.com/robot/send?access_token=c4086d8ba377fdade2dff869e71063733095bc718d3bafdfbe8be0966aa050d6'
    # url = 'https://oapi.dingtalk.com/robot/send?access_token=bee997dbf61e839a17de087830ffef6e864c3109fef62a956703bdfe043b0e10'
    response = requests.post(url=url, data=json_data, headers=headers)
    response.raise_for_status()


# shh非核心业务调用数
def get_shh_non_core_interface_cnt(dt):
    sql = f'''
SELECT sum(cnt) cnt
FROM
  (SELECT count(1) cnt
   FROM ent_raw.interface_base
   WHERE topic = "ent_monitor_interface"
     AND dt = "{dt}"
     AND GET_JSON_OBJECT(ori_json, "$.type") != "EXPORT"
     AND GET_JSON_OBJECT(ori_json, "$.source")= 'CONTACT'
   UNION ALL SELECT count(1) cnt
   FROM ent_raw.interface_base
   WHERE topic = "ent_shh_bizr_interface"
     AND dt = "{dt}"
     AND GET_JSON_OBJECT(ori_json, "$.type") IN("ROOT",
                                                "COMPANY_COUNT")
     AND GET_JSON_OBJECT(ori_json, "$.source")= 'BIZR'
   UNION ALL SELECT count(1) cnt
   FROM ent_raw.interface_base
   WHERE topic = "ent_shh_mecs_interface"
     AND dt = "{dt}"
     AND GET_JSON_OBJECT(ori_json, "$.type") IN("CORP",
                                                "SITE")
     AND GET_JSON_OBJECT(ori_json, "$.source")= 'MECS'
      UNION ALL SELECT count(1) cnt
   FROM ent_raw.interface_base
   WHERE topic = "ent_shh_interface"
     AND dt = "{dt}"
     AND GET_JSON_OBJECT(ori_json, "$.type")= "BIZR"
     AND GET_JSON_OBJECT(ori_json, "$.source")= "BIZR"
     )t
     '''
    return spark.query(sql)[0].collect()[0]['cnt']


def get_shh_company_interface_cnt(dt):
    sql = f'select count(1) cnt from ent_ods.ent_shh_api_company_logs where dt = "{dt}" and source != "SCRIPT"'
    return spark.query(sql)[0].collect()[0]['cnt']


def get_shh_company_interface_script_cnt(dt):
    sql = f'select count(1) cnt from ent_ods.ent_shh_api_company_logs where dt = "{dt}" and source = "SCRIPT"'
    return spark.query(sql)[0].collect()[0]['cnt']


def get_shh_contact_interface_cnt(dt):
    sql = f'select count(1) cnt from ent_raw.interface_base where topic = "ctc_shh_interface" and dt = "{dt}" and GET_JSON_OBJECT(ori_json, "$.source") != "SCRIPT"'
    return spark.query(sql)[0].collect()[0]['cnt']


def get_shh_contact_interface_script_cnt(dt):
    sql = f'select count(1) cnt from ent_raw.interface_base where topic = "ctc_shh_interface" and dt = "{dt}" and GET_JSON_OBJECT(ori_json, "$.source") = "SCRIPT"'
    return spark.query(sql)[0].collect()[0]['cnt']


def get_snv_contact_interface_cnt(dt):
    sql = f'select count(1) cnt from ent_raw.interface_base where topic = "ctc_snovio_interface" and dt = "{dt}" and GET_JSON_OBJECT(ori_json, "$.source") != "MANUAL_CONSUME" '
    return spark.query(sql)[0].collect()[0]['cnt']


def get_snv_contact_interface_script_cnt(dt):
    sql = f'select count(1) cnt from ent_raw.interface_base where topic = "ctc_snovio_interface" and dt = "{dt}" and GET_JSON_OBJECT(ori_json, "$.source") = "MANUAL_CONSUME" '
    return spark.query(sql)[0].collect()[0]['cnt']


if __name__ == '__main__':
    CONFIG, _ = parse_args(sys.argv[1:])
    dts = CONFIG.get('dt').split(',')
    for dt in dts:
        format_dt = f'{dt[:4]}-{dt[4:6]}-{dt[6:]}'
        shh_company_interface_cnt = get_shh_company_interface_cnt(dt)
        shh_company_interface_script_cnt = get_shh_company_interface_script_cnt(dt)
        shh_contact_interface_cnt = get_shh_contact_interface_cnt(dt)
        shh_contact_interface_script_cnt = get_shh_contact_interface_script_cnt(dt)
        snv_contact_interface_cnt = get_snv_contact_interface_cnt(dt)
        snv_contact_interface_script_cnt = get_snv_contact_interface_script_cnt(dt)
        shh_non_core_interface_cnt = get_shh_non_core_interface_cnt(dt)
        msg = f'''【接口调用量统计】------------------------------------------
统计日期: {format_dt}
1、单接口调用公司信息次数: {shh_company_interface_cnt + shh_company_interface_script_cnt}
①自然调用次数: {shh_company_interface_cnt}
②脚本调用次数: {shh_company_interface_script_cnt}

2、单接口调用联系人次数: {shh_contact_interface_cnt + shh_contact_interface_script_cnt}
①自然调用次数: {shh_contact_interface_cnt}
②脚本调用次数: {shh_contact_interface_script_cnt}

3、snovio调用联系人次数: {snv_contact_interface_cnt + snv_contact_interface_script_cnt}
①自然调用次数: {snv_contact_interface_cnt}
②脚本调用次数: {snv_contact_interface_script_cnt}

4、单接口非核心业务调用次数：{shh_non_core_interface_cnt}
---------------------------------------------------------------'''
        print(msg)
        send_dingtalk_notification(msg)