Просмотр исходного кода

feat(ads): 拼团漏斗 DataX 导出 PG(daily 加 funnel_dt 冗余列 + 两 export ini)

daily 分区列 dt 不落 ORC、DataX hdfsreader 导不出,故加冗余普通列 funnel_dt=dt
供导出(funnel_dt→PG dt)。rolling 的 dt 本是普通列直接导。
两 ini 走 datax-hdfs-export-starter,TRUNCATE 全量覆盖到 hs_sync_data.ads。

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
tianyu.chu 1 месяц назад
Родитель
Сommit
cde0c9490c

+ 1 - 0
jobs/ads/trd/ads_trd_group_funnel.sql

@@ -10,6 +10,7 @@
 -- ① 昨日逐日:增量写 dt=${dt} 单分区
 INSERT OVERWRITE TABLE ads.ads_trd_group_funnel_daily PARTITION (dt = '${dt}')
 SELECT
+    '${dt}'                                                                         AS funnel_dt,
     COUNT(DISTINCT CASE WHEN event_name = '$AppStart'            THEN user_id END) AS uv_start,
     COUNT(DISTINCT CASE WHEN event_name = 'GroupShow'            THEN user_id END) AS uv_show,
     COUNT(DISTINCT CASE WHEN event_name = 'ClickGroup'           THEN user_id END) AS uv_detail,

+ 24 - 0
jobs/ads/trd/ads_trd_group_funnel_daily.ini

@@ -0,0 +1,24 @@
+; Hive ads.ads_trd_group_funnel_daily → PG ads.ads_trd_group_funnel_daily
+; 全量覆盖(数据小,每次 TRUNCATE 重灌全部历史分区);funnel_dt(ORC) → dt(PG)
+; 跑法:python3 bin/datax-hdfs-export-starter.py -ini jobs/ads/trd/ads_trd_group_funnel_daily.ini -start-date 20260621 -stop-date 20260621 -random -skip-check
+
+[reader]
+dataSource = hdfs/prd-hdfs-ha
+path = /user/hive/warehouse/ads.db/ads_trd_group_funnel_daily/dt=*/
+column = funnel_dt,uv_start,uv_show,uv_detail,uv_order,uv_paid,etl_time
+columnType =
+fileType = orc
+encoding = UTF-8
+compress =
+nullFormat =
+fieldDelimiter = \t
+
+[writer]
+dataSource = postgresql/dev-poyee-xiaoxu
+database = hs_sync_data
+table = ads.ads_trd_group_funnel_daily
+column = dt,uv_start,uv_show,uv_detail,uv_order,uv_paid,etl_time
+columnType =
+preSql = TRUNCATE TABLE ads.ads_trd_group_funnel_daily
+postSql =
+batchSize = 1024

+ 24 - 0
jobs/ads/trd/ads_trd_group_funnel_rolling.ini

@@ -0,0 +1,24 @@
+; Hive ads.ads_trd_group_funnel_rolling → PG ads.ads_trd_group_funnel_rolling
+; 全量覆盖(非分区单行,TRUNCATE 重灌);dt 是普通 ORC 列直接导
+; 跑法:python3 bin/datax-hdfs-export-starter.py -ini jobs/ads/trd/ads_trd_group_funnel_rolling.ini -start-date 20260621 -stop-date 20260621 -random -skip-check
+
+[reader]
+dataSource = hdfs/prd-hdfs-ha
+path = /user/hive/warehouse/ads.db/ads_trd_group_funnel_rolling
+column = dt,uv_start_7d,uv_show_7d,uv_detail_7d,uv_order_7d,uv_paid_7d,uv_start_30d,uv_show_30d,uv_detail_30d,uv_order_30d,uv_paid_30d,etl_time
+columnType =
+fileType = orc
+encoding = UTF-8
+compress =
+nullFormat =
+fieldDelimiter = \t
+
+[writer]
+dataSource = postgresql/dev-poyee-xiaoxu
+database = hs_sync_data
+table = ads.ads_trd_group_funnel_rolling
+column = dt,uv_start_7d,uv_show_7d,uv_detail_7d,uv_order_7d,uv_paid_7d,uv_start_30d,uv_show_30d,uv_detail_30d,uv_order_30d,uv_paid_30d,etl_time
+columnType =
+preSql = TRUNCATE TABLE ads.ads_trd_group_funnel_rolling
+postSql =
+batchSize = 1024

+ 2 - 0
manual/ddl/ads/trd/ads_trd_group_funnel_create.sql

@@ -9,7 +9,9 @@
 CREATE DATABASE IF NOT EXISTS ads;
 
 -- ① 昨日逐日(增量,分区累积)
+-- funnel_dt = 分区 dt 的冗余普通列:分区列不落 ORC,DataX hdfsreader 导不出,故冗余一列供导 PG
 CREATE TABLE IF NOT EXISTS ads.ads_trd_group_funnel_daily (
+    funnel_dt STRING    COMMENT '快照日 yyyyMMdd(= 分区 dt 冗余列,供 DataX 导 PG)',
     uv_start  BIGINT    COMMENT '启动UV($AppStart,登录用户)',
     uv_show   BIGINT    COMMENT '曝光UV(GroupShow)',
     uv_detail BIGINT    COMMENT '拼团详情UV(ClickGroup)',