Selaa lähdekoodia

feat(manual): 加 raw his_o 历史表 + 配套 backfill ini

tianyu.chu 1 viikko sitten
vanhempi
commit
456994a403

+ 38 - 0
manual/backfill/20260425_raw_trd_card_group_order_info_his_o_history.ini

@@ -0,0 +1,38 @@
+; 作者:tianyu.chu
+; 日期:2026-04-25
+; 工单:(无,项目初建首张正式 raw 表存量回填)
+; 目的:PG public.card_group_order_info → Hive raw.raw_trd_card_group_order_info_his_o
+;       一次性历史回填,用 -backfill 模式按日切分跑
+; 状态:[待执行]
+;
+; 配套 DDL:manual/ddl/raw/trd/raw_trd_card_group_order_info_his_o_create.sql
+; 跑批参考命令(在 m2 / cdhmaster02 上 bigdata 用户):
+;   nohup python3 bin/datax-hive-import-starter.py \
+;     -ini manual/backfill/20260425_raw_trd_card_group_order_info_his_o_history.ini \
+;     -start-date <yyyymmdd> -stop-date <yyyymmdd> \
+;     -random -backfill \
+;     > workspace/backfill_his_o.log 2>&1 &
+;
+; 注:key = value 不对齐
+
+[reader]
+dataSource = postgresql/prd-poyee-aliyun
+database = hobby_stocks
+table = public.card_group_order_info
+column = id,group_info_id,merchant_id,user_id,shipping_address_id,purchase_count,order_no,accounts_payable,actual_payment,payment_type,payment_time,coupon,discount,status,remark,create_time,create_by,update_time,update_by,payment_status,payment_status_desc,payment_success_time,del_flg,curier_company,refund_fee,refund_time,anonymous,pick_up_type,ship_time,refund_success_time,refund_recv_accout,refund_account,refund_request_source,card_price,act_price,goods_price_json,payment_sub_type,team_first,refuse_status,prop1,prop2,prop3,point,order_type,trade_amount,refund_type,refund_reason,evaluation,user_refund_time,refund_status,merchant_refund_reason,point_deduct,shipping_cost,merchant_remark,pay_record,order_sub_type,give_user_code,give_order_id,read_flag,give_num,invoice_id,combination_no,open_self,refund_desc,goods_allocate,close_payment_status,close_payment_time,finished_time,expire_time,settlement_amount,platform_coupon,platform_discount,discount_amount,member_discount,shipping_free_id,shipping_free_amount,discount_point,un_shipped_num,pre_un_shipped_num,wait_shipped_num,pre_wait_shipped_num,refuse_time,refuse_notice,pickup_time,waring_type,waring_status,point_type,delivery_end_time,serve_status,self_pickup_time,act_discount
+columnType =
+where = create_time >= '${start_date}' AND create_time < '${stop_date}'
+querySql =
+splitPk = id
+fetchSize = 1000
+
+[writer]
+dataSource = hdfs/prd-hdfs-ha
+path = /user/hive/warehouse/raw.db/raw_trd_card_group_order_info_his_o/dt=${dt}/
+column = id,group_info_id,merchant_id,user_id,shipping_address_id,purchase_count,order_no,accounts_payable,actual_payment,payment_type,payment_time,coupon,discount,status,remark,create_time,create_by,update_time,update_by,payment_status,payment_status_desc,payment_success_time,del_flg,curier_company,refund_fee,refund_time,anonymous,pick_up_type,ship_time,refund_success_time,refund_recv_accout,refund_account,refund_request_source,card_price,act_price,goods_price_json,payment_sub_type,team_first,refuse_status,prop1,prop2,prop3,point,order_type,trade_amount,refund_type,refund_reason,evaluation,user_refund_time,refund_status,merchant_refund_reason,point_deduct,shipping_cost,merchant_remark,pay_record,order_sub_type,give_user_code,give_order_id,read_flag,give_num,invoice_id,combination_no,open_self,refund_desc,goods_allocate,close_payment_status,close_payment_time,finished_time,expire_time,settlement_amount,platform_coupon,platform_discount,discount_amount,member_discount,shipping_free_id,shipping_free_amount,discount_point,un_shipped_num,pre_un_shipped_num,wait_shipped_num,pre_wait_shipped_num,refuse_time,refuse_notice,pickup_time,waring_type,waring_status,point_type,delivery_end_time,serve_status,self_pickup_time,act_discount
+columnType =
+fileType = orc
+fileName = raw_trd_card_group_order_info_his_o
+encoding = UTF-8
+writeMode = truncate
+fieldDelimiter = \t

+ 106 - 0
manual/ddl/raw/trd/raw_trd_card_group_order_info_his_o_create.sql

@@ -0,0 +1,106 @@
+-- 作者:tianyu.chu
+-- 日期:2026-04-25
+-- 工单:(无,项目初建首张正式 raw 表)
+-- 目的:raw 层贴源历史表,承接 PG public.card_group_order_info 全量历史数据;
+--       91 字段(砍 28 敏感/冗余),按 dt 分区 ORC 存储
+-- 状态:[待执行]
+
+DROP TABLE IF EXISTS raw.raw_trd_card_group_order_info_his_o;
+
+CREATE EXTERNAL TABLE IF NOT EXISTS raw.raw_trd_card_group_order_info_his_o (
+    id                             STRING COMMENT 'id',
+    group_info_id                  STRING COMMENT '组团信息id',
+    merchant_id                    STRING COMMENT '商家id',
+    user_id                        STRING COMMENT '用户id',
+    shipping_address_id            STRING COMMENT '收货地址id',
+    purchase_count                 STRING COMMENT '购买数量',
+    order_no                       STRING COMMENT '订单编码',
+    accounts_payable               STRING COMMENT '应付款',
+    actual_payment                 STRING COMMENT '实付款',
+    payment_type                   STRING COMMENT '支付方式-交易类型',
+    payment_time                   STRING COMMENT '支付时间',
+    coupon                         STRING COMMENT '优惠券',
+    discount                       STRING COMMENT '折扣',
+    status                         STRING COMMENT '订单状态',
+    remark                         STRING COMMENT '备注',
+    create_time                    STRING COMMENT '创建时间',
+    create_by                      STRING COMMENT '创建人',
+    update_time                    STRING COMMENT '更新时间',
+    update_by                      STRING COMMENT '更新人',
+    payment_status                 STRING COMMENT '交易状态',
+    payment_status_desc            STRING COMMENT '交易状态描述',
+    payment_success_time           STRING COMMENT '支付完成时间',
+    del_flg                        STRING COMMENT '删除标记:0=正常,1=删除',
+    curier_company                 STRING COMMENT '快递公司',
+    refund_fee                     STRING COMMENT '退款金额',
+    refund_time                    STRING COMMENT '退款时间',
+    anonymous                      STRING COMMENT '是否匿名',
+    pick_up_type                   STRING COMMENT '提货方式',
+    ship_time                      STRING COMMENT '发货时间',
+    refund_success_time            STRING COMMENT '退款成功时间',
+    refund_recv_accout             STRING COMMENT '退款入账账户',
+    refund_account                 STRING COMMENT '退款资金来源',
+    refund_request_source          STRING COMMENT '退款发起来源',
+    card_price                     STRING COMMENT '应付款',
+    act_price                      STRING COMMENT '应付款',
+    goods_price_json               STRING COMMENT '价格json',
+    payment_sub_type               STRING COMMENT '支付子分类',
+    team_first                     STRING COMMENT '买队优先队伍',
+    refuse_status                  STRING COMMENT '是否接受累积发货,0申请,1拒绝,2同意',
+    prop1                          STRING COMMENT '备用',
+    prop2                          STRING COMMENT '备用',
+    prop3                          STRING COMMENT '备用',
+    point                          STRING COMMENT '消耗积分',
+    order_type                     STRING COMMENT '订单类型',
+    trade_amount                   STRING COMMENT '订单交易金额',
+    refund_type                    STRING COMMENT '退款类型',
+    refund_reason                  STRING COMMENT '订单退换原因',
+    evaluation                     STRING COMMENT '订单评价',
+    user_refund_time               STRING COMMENT '退换申请时间',
+    refund_status                  STRING COMMENT '退款状态',
+    merchant_refund_reason         STRING COMMENT '商家拒绝原因',
+    point_deduct                   STRING COMMENT '积分抵扣金额',
+    shipping_cost                  STRING COMMENT '运费',
+    merchant_remark                STRING COMMENT '商家备注',
+    pay_record                     STRING COMMENT '是否重复支付:1=是',
+    order_sub_type                 STRING COMMENT '订单子类型',
+    give_user_code                 STRING COMMENT '赠与人',
+    give_order_id                  STRING COMMENT '赠与关联订单id',
+    read_flag                      STRING COMMENT '赠送未读0和1',
+    give_num                       STRING COMMENT '赠送个数',
+    invoice_id                     STRING COMMENT '发票记录id',
+    combination_no                 STRING COMMENT '拆分订单关联编号',
+    open_self                      STRING COMMENT '是否用户自己拆卡',
+    refund_desc                    STRING COMMENT '退款原因详细描述',
+    goods_allocate                 STRING COMMENT '卡密是否分配',
+    close_payment_status           STRING COMMENT '打款状态',
+    close_payment_time             STRING COMMENT '打款时间',
+    finished_time                  STRING COMMENT '订单结束时间',
+    expire_time                    STRING COMMENT '过期时间',
+    settlement_amount              STRING COMMENT '结算金额',
+    platform_coupon                STRING COMMENT '平台优惠券id',
+    platform_discount              STRING COMMENT '平台优惠劵折扣',
+    discount_amount                STRING COMMENT '折扣金额',
+    member_discount                STRING COMMENT '会员折扣',
+    shipping_free_id               STRING COMMENT '运费券id',
+    shipping_free_amount           STRING COMMENT '运费券金额',
+    discount_point                 STRING COMMENT '折扣积分',
+    un_shipped_num                 STRING COMMENT '精美卡片未发货数量',
+    pre_un_shipped_num             STRING COMMENT '拼豆订单提醒用户申请时间',
+    wait_shipped_num               STRING COMMENT '精美卡片等待发货数量',
+    pre_wait_shipped_num           STRING COMMENT '用户支付拼豆订单运费时间',
+    refuse_time                    STRING COMMENT '用户同意累计发货时间',
+    refuse_notice                  STRING COMMENT '累计发货通知提醒',
+    pickup_time                    STRING COMMENT '揽收时间',
+    waring_type                    STRING COMMENT '风险异常类型',
+    waring_status                  STRING COMMENT '风险异常状态',
+    point_type                     STRING COMMENT '使用积分类型',
+    delivery_end_time              STRING COMMENT '发货截止时间',
+    serve_status                   STRING COMMENT '订单业务状态',
+    self_pickup_time               STRING COMMENT '申请自提时间,24小时内有效',
+    act_discount                   STRING COMMENT '平台折扣'
+)
+COMMENT '拼团表存量历史截至20260425'
+PARTITIONED BY (dt STRING)
+STORED AS ORC
+LOCATION '/user/hive/warehouse/raw.db/raw_trd_card_group_order_info_his_o';