Parcourir la source

feat(raw/trd): 订单 raw his_o 并入 inc_d + 4月缺口 backfill 配置

his_o(create<20260401) 灌入 inc_d 的一次性 SQL(91列显式 + DISTRIBUTE BY dt);
新增 gapfill ini 补 4/1~4/22 create 窗,inc_d 现有增量分区原地保留;
his_o 建表 DDL 加弃用注释。

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
tianyu.chu il y a 3 semaines
Parent
commit
bc4e707b54

+ 33 - 0
manual/backfill/20260714_raw_trd_card_group_order_info_inc_d_gapfill.ini

@@ -0,0 +1,33 @@
+; 作者:tianyu.chu
+; 日期:2026-07-14
+; 工单:(无)
+; 目的:PG public.card_group_order_info → Hive raw.raw_trd_card_group_order_info_inc_d
+;       合表补缺:his_o(create<20260401) 并入 inc_d 后,补 4/1~4/22 create 窗缺口(-backfill 按日切)
+; 状态:[待执行]
+;
+; 配套 DDL:manual/ddl/raw/trd/raw_trd_card_group_order_info_inc_d_create.sql
+; 配套 SQL:manual/backfill/20260714_raw_trd_card_group_order_info_inc_d_his_merge.sql
+;
+; 说明:create_time 锚点与 his_o 存量口径一致;实际 range 以 SHOW PARTITIONS 探查的空分区为准
+
+[reader]
+dataSource = postgresql/prd-poyee-aliyun
+database = hobby_stocks
+table = public.card_group_order_info
+column = id,group_info_id,merchant_id,user_id,shipping_address_id,purchase_count,order_no,accounts_payable,actual_payment,payment_type,payment_time,coupon,discount,status,remark,create_time,create_by,update_time,update_by,payment_status,payment_status_desc,payment_success_time,del_flg,curier_company,refund_fee,refund_time,anonymous,pick_up_type,ship_time,refund_success_time,refund_recv_accout,refund_account,refund_request_source,card_price,act_price,goods_price_json,payment_sub_type,team_first,refuse_status,prop1,prop2,prop3,point,order_type,trade_amount,refund_type,refund_reason,evaluation,user_refund_time,refund_status,merchant_refund_reason,point_deduct,shipping_cost,merchant_remark,pay_record,order_sub_type,give_user_code,give_order_id,read_flag,give_num,invoice_id,combination_no,open_self,refund_desc,goods_allocate,close_payment_status,close_payment_time,finished_time,expire_time,settlement_amount,platform_coupon,platform_discount,discount_amount,member_discount,shipping_free_id,shipping_free_amount,discount_point,un_shipped_num,pre_un_shipped_num,wait_shipped_num,pre_wait_shipped_num,refuse_time,refuse_notice,pickup_time,waring_type,waring_status,point_type,delivery_end_time,serve_status,self_pickup_time,act_discount
+columnType =
+where = create_time >= '${start_date}' AND create_time < '${stop_date}'
+querySql =
+splitPk = id
+fetchSize = 1000
+
+[writer]
+dataSource = hdfs/prd-hdfs-ha
+path = /user/hive/warehouse/raw.db/raw_trd_card_group_order_info_inc_d/dt=${dt}/
+column = id,group_info_id,merchant_id,user_id,shipping_address_id,purchase_count,order_no,accounts_payable,actual_payment,payment_type,payment_time,coupon,discount,status,remark,create_time,create_by,update_time,update_by,payment_status,payment_status_desc,payment_success_time,del_flg,curier_company,refund_fee,refund_time,anonymous,pick_up_type,ship_time,refund_success_time,refund_recv_accout,refund_account,refund_request_source,card_price,act_price,goods_price_json,payment_sub_type,team_first,refuse_status,prop1,prop2,prop3,point,order_type,trade_amount,refund_type,refund_reason,evaluation,user_refund_time,refund_status,merchant_refund_reason,point_deduct,shipping_cost,merchant_remark,pay_record,order_sub_type,give_user_code,give_order_id,read_flag,give_num,invoice_id,combination_no,open_self,refund_desc,goods_allocate,close_payment_status,close_payment_time,finished_time,expire_time,settlement_amount,platform_coupon,platform_discount,discount_amount,member_discount,shipping_free_id,shipping_free_amount,discount_point,un_shipped_num,pre_un_shipped_num,wait_shipped_num,pre_wait_shipped_num,refuse_time,refuse_notice,pickup_time,waring_type,waring_status,point_type,delivery_end_time,serve_status,self_pickup_time,act_discount
+columnType =
+fileType = orc
+fileName = raw_trd_card_group_order_info_inc_d
+encoding = UTF-8
+writeMode = truncate
+fieldDelimiter = \t

+ 34 - 0
manual/backfill/20260714_raw_trd_card_group_order_info_inc_d_his_merge.sql

@@ -0,0 +1,34 @@
+-- 作者:tianyu.chu
+-- 日期:2026-07-14
+-- 工单:(无)
+-- 目的:his_o(create_time < 20260401) 一次性并入 inc_d;原地增补,不动 inc_d 现有 4/23+ 增量分区
+-- 状态:[待执行]
+-- 备注:两表 schema 逐字段一致(91 列 STRING);INSERT OVERWRITE 动态分区只覆盖 SELECT 出现的 dt(均 < 20260401),
+--       不触碰 inc_d 的日常增量分区;DISTRIBUTE BY dt 防 ~1600 个老分区写入炸小文件(ADR-15)。
+--       4/1~4/22 create 窗缺口由 gapfill ini 从 PG 补(见同目录 20260714_..._gapfill.ini)。
+
+SET hive.exec.dynamic.partition=true;
+SET hive.exec.dynamic.partition.mode=nonstrict;
+
+INSERT OVERWRITE TABLE raw.raw_trd_card_group_order_info_inc_d PARTITION (dt)
+SELECT
+    id, group_info_id, merchant_id, user_id, shipping_address_id, purchase_count,
+    order_no, accounts_payable, actual_payment, payment_type, payment_time, coupon,
+    discount, status, remark, create_time, create_by, update_time, update_by,
+    payment_status, payment_status_desc, payment_success_time, del_flg, curier_company,
+    refund_fee, refund_time, anonymous, pick_up_type, ship_time, refund_success_time,
+    refund_recv_accout, refund_account, refund_request_source, card_price, act_price,
+    goods_price_json, payment_sub_type, team_first, refuse_status, prop1, prop2, prop3,
+    point, order_type, trade_amount, refund_type, refund_reason, evaluation,
+    user_refund_time, refund_status, merchant_refund_reason, point_deduct, shipping_cost,
+    merchant_remark, pay_record, order_sub_type, give_user_code, give_order_id, read_flag,
+    give_num, invoice_id, combination_no, open_self, refund_desc, goods_allocate,
+    close_payment_status, close_payment_time, finished_time, expire_time, settlement_amount,
+    platform_coupon, platform_discount, discount_amount, member_discount, shipping_free_id,
+    shipping_free_amount, discount_point, un_shipped_num, pre_un_shipped_num, wait_shipped_num,
+    pre_wait_shipped_num, refuse_time, refuse_notice, pickup_time, waring_type, waring_status,
+    point_type, delivery_end_time, serve_status, self_pickup_time, act_discount,
+    dt
+FROM raw.raw_trd_card_group_order_info_his_o
+WHERE dt < '20260401'
+DISTRIBUTE BY dt;

+ 1 - 1
manual/ddl/raw/trd/raw_trd_card_group_order_info_his_o_create.sql

@@ -3,7 +3,7 @@
 -- 工单:(无,项目初建首张正式 raw 表)
 -- 目的:raw 层贴源历史表,承接 PG public.card_group_order_info 全量历史数据;
 --       91 字段(砍 28 敏感/冗余),按 dt 分区 ORC 存储
--- 状态:[已执行]
+-- 状态:[已弃用] 2026-07-14 表数据(create<20260401) 并入 raw_trd_card_group_order_info_inc_d,his_o 表已 DROP;本 DDL 仅留档
 -- 备注:拼团表存量历史以create_time为基准字段截至20260425(不包含),下一步做增量使用update_time从20260323开始至今,留两天重叠窗口。
 
 DROP TABLE IF EXISTS raw.raw_trd_card_group_order_info_his_o;