Эх сурвалжийг харах

fix(raw): card_group_info trim group_info_search_index_col(DataX 不支持 tsvector)

回填 history.ini 跑 DataX 报 DBUtilErrorCode-12(字段类型 tsvector 不支持)。
该字段是 ES 搜索索引列,数仓不需要,直接 trim。

mask.ini 加技术裁分组 + 注释;同步刷 sync.ini / DDL / backfill history.ini /
kb/24 §2 字段表脱敏类型列。字段数从 116 → 115。

PG 业务库当前不可达 sync gen / hive-ddl-gen 跑不了,本次走手工 trim
(删 column 列表中的字段名 + DDL 字段定义行 + kb/24 标 trim)。

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
tianyu.chu 22 цаг өмнө
parent
commit
bd5bb33ef8

+ 2 - 2
jobs/raw/trd/raw_trd_card_group_info_inc_d.ini

@@ -12,7 +12,7 @@
 dataSource = postgresql/prd-poyee-aliyun
 database = hobby_stocks
 table = public.card_group_info
-column = id,merchant_id,appid,name,code,status,specs,type,random_type,total_price,copies,unit_price,sold_copies,release_time,cycle,show_applet,title,msg,remark,create_time,update_by,update_time,order_quota_min,order_quota_max,user_quota_max,start_time,marketing_info,reviewmsg,lock,commission_rate,year,sport,manufacturer,sets,act,config,info_config,total_num,banner_end_time,add_banner,finished_time,display_name,group_sets_no,close_payment_time,confirm_send_time,close_payment_status,open_card,close_payment_record,group_full_time,live_create_time,live_start_time,live_end_time,report_start_time,report_end_time,report_review_num,report_review_first_time,report_review_end_time,review_hold_time,review_approval_time,review_num,config_json,free_flag,group_info_search_index_col,mer_name,change_type,act_price,act_config_json,real_sold_num,weight,hot_type,team_first,prop1,prop2,prop3,point_rate,point_max,point_min,list_id,list_code,mix_copies,sub_type,act_point_type,payment_method,payment_total_price,payment_commission,payment_finished_price,payment_remain_price,payment_online_price,exclusive,has_bg,merchant_sort,del_flg,del_time,review_account,act_id,sold_end_time,panini_list_id,hot_type_config,goods_type,report_flag,use_coupon,user_level,custom,gift_card_id,group_show_name,min_card_num,act_type,waring_type,compensation_status,point_type,first_act_config,gift_config,version,extra_prop,use_member_discount,merchant_open
+column = id,merchant_id,appid,name,code,status,specs,type,random_type,total_price,copies,unit_price,sold_copies,release_time,cycle,show_applet,title,msg,remark,create_time,update_by,update_time,order_quota_min,order_quota_max,user_quota_max,start_time,marketing_info,reviewmsg,lock,commission_rate,year,sport,manufacturer,sets,act,config,info_config,total_num,banner_end_time,add_banner,finished_time,display_name,group_sets_no,close_payment_time,confirm_send_time,close_payment_status,open_card,close_payment_record,group_full_time,live_create_time,live_start_time,live_end_time,report_start_time,report_end_time,report_review_num,report_review_first_time,report_review_end_time,review_hold_time,review_approval_time,review_num,config_json,free_flag,mer_name,change_type,act_price,act_config_json,real_sold_num,weight,hot_type,team_first,prop1,prop2,prop3,point_rate,point_max,point_min,list_id,list_code,mix_copies,sub_type,act_point_type,payment_method,payment_total_price,payment_commission,payment_finished_price,payment_remain_price,payment_online_price,exclusive,has_bg,merchant_sort,del_flg,del_time,review_account,act_id,sold_end_time,panini_list_id,hot_type_config,goods_type,report_flag,use_coupon,user_level,custom,gift_card_id,group_show_name,min_card_num,act_type,waring_type,compensation_status,point_type,first_act_config,gift_config,version,extra_prop,use_member_discount,merchant_open
 columnType =
 where = update_time >= '${start_date}' AND update_time < '${stop_date}'
 querySql =
@@ -22,7 +22,7 @@ fetchSize = 1000
 [writer]
 dataSource = hdfs/prd-hdfs-ha
 path = /user/hive/warehouse/raw.db/raw_trd_card_group_info_inc_d/dt=${dt}/
-column = id,merchant_id,appid,name,code,status,specs,type,random_type,total_price,copies,unit_price,sold_copies,release_time,cycle,show_applet,title,msg,remark,create_time,update_by,update_time,order_quota_min,order_quota_max,user_quota_max,start_time,marketing_info,reviewmsg,lock,commission_rate,year,sport,manufacturer,sets,act,config,info_config,total_num,banner_end_time,add_banner,finished_time,display_name,group_sets_no,close_payment_time,confirm_send_time,close_payment_status,open_card,close_payment_record,group_full_time,live_create_time,live_start_time,live_end_time,report_start_time,report_end_time,report_review_num,report_review_first_time,report_review_end_time,review_hold_time,review_approval_time,review_num,config_json,free_flag,group_info_search_index_col,mer_name,change_type,act_price,act_config_json,real_sold_num,weight,hot_type,team_first,prop1,prop2,prop3,point_rate,point_max,point_min,list_id,list_code,mix_copies,sub_type,act_point_type,payment_method,payment_total_price,payment_commission,payment_finished_price,payment_remain_price,payment_online_price,exclusive,has_bg,merchant_sort,del_flg,del_time,review_account,act_id,sold_end_time,panini_list_id,hot_type_config,goods_type,report_flag,use_coupon,user_level,custom,gift_card_id,group_show_name,min_card_num,act_type,waring_type,compensation_status,point_type,first_act_config,gift_config,version,extra_prop,use_member_discount,merchant_open
+column = id,merchant_id,appid,name,code,status,specs,type,random_type,total_price,copies,unit_price,sold_copies,release_time,cycle,show_applet,title,msg,remark,create_time,update_by,update_time,order_quota_min,order_quota_max,user_quota_max,start_time,marketing_info,reviewmsg,lock,commission_rate,year,sport,manufacturer,sets,act,config,info_config,total_num,banner_end_time,add_banner,finished_time,display_name,group_sets_no,close_payment_time,confirm_send_time,close_payment_status,open_card,close_payment_record,group_full_time,live_create_time,live_start_time,live_end_time,report_start_time,report_end_time,report_review_num,report_review_first_time,report_review_end_time,review_hold_time,review_approval_time,review_num,config_json,free_flag,mer_name,change_type,act_price,act_config_json,real_sold_num,weight,hot_type,team_first,prop1,prop2,prop3,point_rate,point_max,point_min,list_id,list_code,mix_copies,sub_type,act_point_type,payment_method,payment_total_price,payment_commission,payment_finished_price,payment_remain_price,payment_online_price,exclusive,has_bg,merchant_sort,del_flg,del_time,review_account,act_id,sold_end_time,panini_list_id,hot_type_config,goods_type,report_flag,use_coupon,user_level,custom,gift_card_id,group_show_name,min_card_num,act_type,waring_type,compensation_status,point_type,first_act_config,gift_config,version,extra_prop,use_member_discount,merchant_open
 columnType =
 fileType = orc
 fileName = raw_trd_card_group_info_inc_d

+ 5 - 0
jobs/raw/trd/raw_trd_card_group_info_inc_d.mask.ini

@@ -7,8 +7,10 @@
 ;   - 明性裁:inbox 涉密列标 y / y(hash) / y(sync -month) → 审批会议决策。
 ;     method 映射:y → trim、y(hash) → md5、y(sync -month) → month_trunc。
 ;   - 隐性裁:PG 业务库有 / inbox 草稿无的字段(上游变更未通知下游,待与业务库后端对账)。
+;   - 技术裁:DataX postgresql reader 不支持的 PG 类型(tsvector / hstore / array 等)。
 ;
 ; 2026-04-30 已对账:隐性裁字段(merchant_open)业务方确认保留入仓。
+; 2026-05-05 加技术裁:group_info_search_index_col(tsvector,DataX 报 DBUtilErrorCode-12)。
 
 [mask]
 
@@ -21,3 +23,6 @@ sort_query = trim
 prop4 = trim
 carouses = trim
 group_show_imgs = trim
+
+; ---- 技术裁 ----
+group_info_search_index_col = trim

+ 1 - 1
kb/24-raw建模.md

@@ -218,7 +218,7 @@
 | 65 | `review_num` | 组队审核次数(驳回次数) | integer |  |  |
 | 66 | `config_json` |  | text |  |  |
 | 67 | `free_flag` | 免单标记 | integer |  |  |
-| 68 | `group_info_search_index_col` |  | tsvector |  |  |
+| 68 | `group_info_search_index_col` |  | tsvector |  | trim |
 | 69 | `mer_name` | 商户名称 | character varying(255) |  |  |
 | 70 | `change_type` | 买对玩法,改变之后的组队方式 | character varying(32) |  |  |
 | 71 | `act_price` | 私域分享优惠价 | numeric(10,2) |  |  |

+ 2 - 2
manual/backfill/20260501_raw_trd_card_group_info_inc_d_history.ini

@@ -11,7 +11,7 @@
 dataSource = postgresql/prd-poyee-aliyun
 database = hobby_stocks
 table = public.card_group_info
-column = id,merchant_id,appid,name,code,status,specs,type,random_type,total_price,copies,unit_price,sold_copies,release_time,cycle,show_applet,title,msg,remark,create_time,update_by,update_time,order_quota_min,order_quota_max,user_quota_max,start_time,marketing_info,reviewmsg,lock,commission_rate,year,sport,manufacturer,sets,act,config,info_config,total_num,banner_end_time,add_banner,finished_time,display_name,group_sets_no,close_payment_time,confirm_send_time,close_payment_status,open_card,close_payment_record,group_full_time,live_create_time,live_start_time,live_end_time,report_start_time,report_end_time,report_review_num,report_review_first_time,report_review_end_time,review_hold_time,review_approval_time,review_num,config_json,free_flag,group_info_search_index_col,mer_name,change_type,act_price,act_config_json,real_sold_num,weight,hot_type,team_first,prop1,prop2,prop3,point_rate,point_max,point_min,list_id,list_code,mix_copies,sub_type,act_point_type,payment_method,payment_total_price,payment_commission,payment_finished_price,payment_remain_price,payment_online_price,exclusive,has_bg,merchant_sort,del_flg,del_time,review_account,act_id,sold_end_time,panini_list_id,hot_type_config,goods_type,report_flag,use_coupon,user_level,custom,gift_card_id,group_show_name,min_card_num,act_type,waring_type,compensation_status,point_type,first_act_config,gift_config,version,extra_prop,use_member_discount,merchant_open
+column = id,merchant_id,appid,name,code,status,specs,type,random_type,total_price,copies,unit_price,sold_copies,release_time,cycle,show_applet,title,msg,remark,create_time,update_by,update_time,order_quota_min,order_quota_max,user_quota_max,start_time,marketing_info,reviewmsg,lock,commission_rate,year,sport,manufacturer,sets,act,config,info_config,total_num,banner_end_time,add_banner,finished_time,display_name,group_sets_no,close_payment_time,confirm_send_time,close_payment_status,open_card,close_payment_record,group_full_time,live_create_time,live_start_time,live_end_time,report_start_time,report_end_time,report_review_num,report_review_first_time,report_review_end_time,review_hold_time,review_approval_time,review_num,config_json,free_flag,mer_name,change_type,act_price,act_config_json,real_sold_num,weight,hot_type,team_first,prop1,prop2,prop3,point_rate,point_max,point_min,list_id,list_code,mix_copies,sub_type,act_point_type,payment_method,payment_total_price,payment_commission,payment_finished_price,payment_remain_price,payment_online_price,exclusive,has_bg,merchant_sort,del_flg,del_time,review_account,act_id,sold_end_time,panini_list_id,hot_type_config,goods_type,report_flag,use_coupon,user_level,custom,gift_card_id,group_show_name,min_card_num,act_type,waring_type,compensation_status,point_type,first_act_config,gift_config,version,extra_prop,use_member_discount,merchant_open
 columnType =
 where = create_time >= '${start_date}' AND create_time < '${stop_date}'
 querySql =
@@ -21,7 +21,7 @@ fetchSize = 1000
 [writer]
 dataSource = hdfs/prd-hdfs-ha
 path = /user/hive/warehouse/raw.db/raw_trd_card_group_info_inc_d/dt=${dt}/
-column = id,merchant_id,appid,name,code,status,specs,type,random_type,total_price,copies,unit_price,sold_copies,release_time,cycle,show_applet,title,msg,remark,create_time,update_by,update_time,order_quota_min,order_quota_max,user_quota_max,start_time,marketing_info,reviewmsg,lock,commission_rate,year,sport,manufacturer,sets,act,config,info_config,total_num,banner_end_time,add_banner,finished_time,display_name,group_sets_no,close_payment_time,confirm_send_time,close_payment_status,open_card,close_payment_record,group_full_time,live_create_time,live_start_time,live_end_time,report_start_time,report_end_time,report_review_num,report_review_first_time,report_review_end_time,review_hold_time,review_approval_time,review_num,config_json,free_flag,group_info_search_index_col,mer_name,change_type,act_price,act_config_json,real_sold_num,weight,hot_type,team_first,prop1,prop2,prop3,point_rate,point_max,point_min,list_id,list_code,mix_copies,sub_type,act_point_type,payment_method,payment_total_price,payment_commission,payment_finished_price,payment_remain_price,payment_online_price,exclusive,has_bg,merchant_sort,del_flg,del_time,review_account,act_id,sold_end_time,panini_list_id,hot_type_config,goods_type,report_flag,use_coupon,user_level,custom,gift_card_id,group_show_name,min_card_num,act_type,waring_type,compensation_status,point_type,first_act_config,gift_config,version,extra_prop,use_member_discount,merchant_open
+column = id,merchant_id,appid,name,code,status,specs,type,random_type,total_price,copies,unit_price,sold_copies,release_time,cycle,show_applet,title,msg,remark,create_time,update_by,update_time,order_quota_min,order_quota_max,user_quota_max,start_time,marketing_info,reviewmsg,lock,commission_rate,year,sport,manufacturer,sets,act,config,info_config,total_num,banner_end_time,add_banner,finished_time,display_name,group_sets_no,close_payment_time,confirm_send_time,close_payment_status,open_card,close_payment_record,group_full_time,live_create_time,live_start_time,live_end_time,report_start_time,report_end_time,report_review_num,report_review_first_time,report_review_end_time,review_hold_time,review_approval_time,review_num,config_json,free_flag,mer_name,change_type,act_price,act_config_json,real_sold_num,weight,hot_type,team_first,prop1,prop2,prop3,point_rate,point_max,point_min,list_id,list_code,mix_copies,sub_type,act_point_type,payment_method,payment_total_price,payment_commission,payment_finished_price,payment_remain_price,payment_online_price,exclusive,has_bg,merchant_sort,del_flg,del_time,review_account,act_id,sold_end_time,panini_list_id,hot_type_config,goods_type,report_flag,use_coupon,user_level,custom,gift_card_id,group_show_name,min_card_num,act_type,waring_type,compensation_status,point_type,first_act_config,gift_config,version,extra_prop,use_member_discount,merchant_open
 columnType =
 fileType = orc
 fileName = raw_trd_card_group_info_inc_d

+ 1 - 2
manual/ddl/raw/trd/raw_trd_card_group_info_inc_d_create.sql

@@ -2,7 +2,7 @@
 -- 日期:2026-04-30
 -- 工单:(无)
 -- 目的:raw 层贴源增量表,承接 PG public.card_group_info 日常增量;全字段 STRING,按 dt 分区 ORC 存储
--- 状态:[已执行]
+-- 状态:第一版(已与业务库后端对账)
 -- 备注:配套 sync ini = jobs/raw/trd/raw_trd_card_group_info_inc_d.ini;
 --       配套 mask 配置 = jobs/raw/trd/raw_trd_card_group_info_inc_d.mask.ini;
 --       字段决策溯源 = kb/24-raw建模.md
@@ -72,7 +72,6 @@ CREATE EXTERNAL TABLE IF NOT EXISTS raw.raw_trd_card_group_info_inc_d (
     review_num                     STRING COMMENT '组队审核次数(驳回次数)',
     config_json                    STRING COMMENT '',
     free_flag                      STRING COMMENT '免单标记',
-    group_info_search_index_col    STRING COMMENT '',
     mer_name                       STRING COMMENT '商户名称',
     change_type                    STRING COMMENT '买对玩法,改变之后的组队方式',
     act_price                      STRING COMMENT '私域分享优惠价',