|
|
@@ -0,0 +1,126 @@
|
|
|
+-- 作者：tianyu.chu
|
|
|
+-- 日期：2026-07-23
|
|
|
+-- 工单：(无)
|
|
|
+-- 目的：全量 re-init 重灌——读 dump0722(全表当前态)，按活动日重灌 ods_inc 的全部历史分区（至 0721）。
|
|
|
+-- 每单落两份：创建日 + 活动日；活动日 = GREATEST(create, update)（兼容后端 update<create 脏单，取较晚者）。
|
|
|
+-- 状态：[待执行]
|
|
|
+-- 备注：口径与 jobs/ods/trd/ods_trd_card_group_order_info_inc_d.sql 完全一致（GREATEST 活动日 + explode 双份）；
|
|
|
+-- 上界 ≤ 0721：0722 交调度自愈（任务日 0723 处理 0722），故 act_dt 封顶 0721、无下界（全史 2021~0721）；
|
|
|
+-- 全史约 1700 个日分区，max.dynamic.partitions/pernode 调 10000；DISTRIBUTE BY dt 防每分区小文件；
|
|
|
+-- explode 用 CASE 去重「创建日=活动日」的单；dump 一单一行，(id,act_dt) 天然不重；
|
|
|
+-- 跑前需先 DROP + 重建空表 ods.ods_trd_card_group_order_info_inc_d（外部表 DROP 不清 HDFS，另跑 hdfs rm）；
|
|
|
+-- 跑完并和 PG 按 create 验平后，dump0722 中转表可 DROP。
|
|
|
+-- 跑法（一次）：python3 bin/spark-sql-starter.py -f manual/backfill/20260723_ods_trd_card_group_order_info_reinit_rebucket.sql
|
|
|
+
|
|
|
+SET hive.exec.dynamic.partition=true;
|
|
|
+SET hive.exec.dynamic.partition.mode=nonstrict;
|
|
|
+SET hive.exec.max.dynamic.partitions=10000;
|
|
|
+SET hive.exec.max.dynamic.partitions.pernode=10000;
|
|
|
+
|
|
|
+INSERT OVERWRITE TABLE ods.ods_trd_card_group_order_info_inc_d PARTITION (dt)
|
|
|
+SELECT
|
|
|
+ CAST(id AS BIGINT) AS id,
|
|
|
+ CAST(group_info_id AS BIGINT) AS group_info_id,
|
|
|
+ CAST(merchant_id AS BIGINT) AS merchant_id,
|
|
|
+ CAST(user_id AS BIGINT) AS user_id,
|
|
|
+ CAST(shipping_address_id AS BIGINT) AS shipping_address_id,
|
|
|
+ CAST(purchase_count AS BIGINT) AS purchase_count,
|
|
|
+ order_no AS order_no,
|
|
|
+ CAST(accounts_payable AS DECIMAL(20,4)) AS accounts_payable,
|
|
|
+ CAST(actual_payment AS DECIMAL(20,4)) AS actual_payment,
|
|
|
+ payment_type AS payment_type,
|
|
|
+ CAST(payment_time AS TIMESTAMP) AS payment_time,
|
|
|
+ CAST(coupon AS BIGINT) AS coupon,
|
|
|
+ CAST(discount AS DECIMAL(20,4)) AS discount,
|
|
|
+ CAST(status AS BIGINT) AS status,
|
|
|
+ remark AS remark,
|
|
|
+ CAST(create_time AS TIMESTAMP) AS create_time,
|
|
|
+ create_by AS create_by,
|
|
|
+ CAST(update_time AS TIMESTAMP) AS update_time,
|
|
|
+ update_by AS update_by,
|
|
|
+ payment_status AS payment_status,
|
|
|
+ payment_status_desc AS payment_status_desc,
|
|
|
+ CAST(payment_success_time AS TIMESTAMP) AS payment_success_time,
|
|
|
+ CAST(del_flg AS BIGINT) AS del_flg,
|
|
|
+ curier_company AS curier_company,
|
|
|
+ CAST(refund_fee AS DECIMAL(20,4)) AS refund_fee,
|
|
|
+ CAST(refund_time AS TIMESTAMP) AS refund_time,
|
|
|
+ CAST(anonymous AS BOOLEAN) AS anonymous,
|
|
|
+ pick_up_type AS pick_up_type,
|
|
|
+ CAST(ship_time AS TIMESTAMP) AS ship_time,
|
|
|
+ CAST(refund_success_time AS TIMESTAMP) AS refund_success_time,
|
|
|
+ refund_recv_accout AS refund_recv_accout,
|
|
|
+ refund_account AS refund_account,
|
|
|
+ refund_request_source AS refund_request_source,
|
|
|
+ CAST(card_price AS DECIMAL(20,4)) AS card_price,
|
|
|
+ CAST(act_price AS DECIMAL(20,4)) AS act_price,
|
|
|
+ goods_price_json AS goods_price_json,
|
|
|
+ payment_sub_type AS payment_sub_type,
|
|
|
+ team_first AS team_first,
|
|
|
+ CAST(refuse_status AS BIGINT) AS refuse_status,
|
|
|
+ prop1 AS prop1,
|
|
|
+ prop2 AS prop2,
|
|
|
+ prop3 AS prop3,
|
|
|
+ CAST(point AS BIGINT) AS point,
|
|
|
+ order_type AS order_type,
|
|
|
+ CAST(trade_amount AS DECIMAL(20,4)) AS trade_amount,
|
|
|
+ refund_type AS refund_type,
|
|
|
+ refund_reason AS refund_reason,
|
|
|
+ evaluation AS evaluation,
|
|
|
+ CAST(user_refund_time AS TIMESTAMP) AS user_refund_time,
|
|
|
+ CAST(refund_status AS BIGINT) AS refund_status,
|
|
|
+ merchant_refund_reason AS merchant_refund_reason,
|
|
|
+ CAST(point_deduct AS DECIMAL(20,4)) AS point_deduct,
|
|
|
+ CAST(shipping_cost AS DECIMAL(20,4)) AS shipping_cost,
|
|
|
+ merchant_remark AS merchant_remark,
|
|
|
+ CAST(pay_record AS BIGINT) AS pay_record,
|
|
|
+ order_sub_type AS order_sub_type,
|
|
|
+ give_user_code AS give_user_code,
|
|
|
+ CAST(give_order_id AS BIGINT) AS give_order_id,
|
|
|
+ CAST(read_flag AS BIGINT) AS read_flag,
|
|
|
+ CAST(give_num AS BIGINT) AS give_num,
|
|
|
+ CAST(invoice_id AS BIGINT) AS invoice_id,
|
|
|
+ combination_no AS combination_no,
|
|
|
+ CAST(open_self AS BIGINT) AS open_self,
|
|
|
+ refund_desc AS refund_desc,
|
|
|
+ CAST(goods_allocate AS BIGINT) AS goods_allocate,
|
|
|
+ CAST(close_payment_status AS BIGINT) AS close_payment_status,
|
|
|
+ CAST(close_payment_time AS TIMESTAMP) AS close_payment_time,
|
|
|
+ CAST(finished_time AS TIMESTAMP) AS finished_time,
|
|
|
+ CAST(expire_time AS TIMESTAMP) AS expire_time,
|
|
|
+ CAST(settlement_amount AS DECIMAL(20,4)) AS settlement_amount,
|
|
|
+ CAST(platform_coupon AS BIGINT) AS platform_coupon,
|
|
|
+ CAST(platform_discount AS DECIMAL(20,4)) AS platform_discount,
|
|
|
+ CAST(discount_amount AS DECIMAL(20,4)) AS discount_amount,
|
|
|
+ CAST(member_discount AS DECIMAL(20,4)) AS member_discount,
|
|
|
+ CAST(shipping_free_id AS BIGINT) AS shipping_free_id,
|
|
|
+ CAST(shipping_free_amount AS DECIMAL(20,4)) AS shipping_free_amount,
|
|
|
+ CAST(discount_point AS BIGINT) AS discount_point,
|
|
|
+ CAST(un_shipped_num AS BIGINT) AS un_shipped_num,
|
|
|
+ CAST(pre_un_shipped_num AS BIGINT) AS pre_un_shipped_num,
|
|
|
+ CAST(wait_shipped_num AS BIGINT) AS wait_shipped_num,
|
|
|
+ CAST(pre_wait_shipped_num AS BIGINT) AS pre_wait_shipped_num,
|
|
|
+ CAST(refuse_time AS TIMESTAMP) AS refuse_time,
|
|
|
+ CAST(refuse_notice AS BIGINT) AS refuse_notice,
|
|
|
+ CAST(pickup_time AS TIMESTAMP) AS pickup_time,
|
|
|
+ waring_type AS waring_type,
|
|
|
+ CAST(waring_status AS BIGINT) AS waring_status,
|
|
|
+ point_type AS point_type,
|
|
|
+ CAST(delivery_end_time AS TIMESTAMP) AS delivery_end_time,
|
|
|
+ CAST(serve_status AS BIGINT) AS serve_status,
|
|
|
+ CAST(self_pickup_time AS TIMESTAMP) AS self_pickup_time,
|
|
|
+ CAST(act_discount AS DECIMAL(20,4)) AS act_discount,
|
|
|
+ CASE WHEN del_flg = '1' THEN TRUE ELSE FALSE END AS is_deleted,
|
|
|
+ act_dt AS dt
|
|
|
+FROM (
|
|
|
+ SELECT *,
|
|
|
+ DATE_FORMAT(create_time, 'yyyyMMdd') AS cdt, -- 创建日
|
|
|
+ DATE_FORMAT(GREATEST(create_time, NULLIF(update_time, '')), 'yyyyMMdd') AS adt -- 活动日 = MAX(create,update)
|
|
|
+ FROM raw.tmp_trd_card_group_order_info_dump0722
|
|
|
+ WHERE dt = '20260722'
|
|
|
+) t
|
|
|
+LATERAL VIEW explode(
|
|
|
+ CASE WHEN cdt = adt THEN array(cdt) ELSE array(cdt, adt) END -- 同日一份；跨日炸成 创建日+活动日 两行
|
|
|
+) e AS act_dt
|
|
|
+WHERE act_dt <= '20260721'
|
|
|
+DISTRIBUTE BY dt;
|
