Explorar el Código

fix(jobs/dim): WITH CTE 改到 INSERT OVERWRITE 之前 - Spark 2.4 语法约束

Spark 2.4 ParseException:INSERT OVERWRITE ... PARTITION (...) WITH cte AS (...)
SELECT 的写法不支持,标准写法是 WITH cte AS (...) INSERT OVERWRITE ... SELECT。
dim_usr_user_ful_d / dim_trd_card_group_ful_d 两文件同改。

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
tianyu.chu hace 3 semanas
padre
commit
6f28cd33b1

+ 1 - 1
jobs/dim/trd/dim_trd_card_group_ful_d.sql

@@ -13,7 +13,6 @@
 --      性能:dim 大表只 scan + broadcast NOT IN 剪枝,不参与 shuffle;
 --      前置 DS DEPENDENT:ods.cgi.dt=${dt} + dim.dt=${pdt}
 
-INSERT OVERWRITE TABLE dim.dim_trd_card_group_ful_d PARTITION (dt='${dt}')
 WITH today_cgi AS (
     SELECT *
     FROM (
@@ -138,6 +137,7 @@ today_rebuilt AS (
         CURRENT_TIMESTAMP()                AS etl_time
     FROM today_cgi cg
 )
+INSERT OVERWRITE TABLE dim.dim_trd_card_group_ful_d PARTITION (dt='${dt}')
 SELECT
     group_info_id,
     code,

+ 1 - 1
jobs/dim/usr/dim_usr_user_ful_d.sql

@@ -13,7 +13,6 @@
 --      性能:dim 大表只 scan + broadcast NOT IN 剪枝,不参与 shuffle;today_changed user 数量小 broadcast 有效;
 --      前置 DS DEPENDENT:ods.base_user.dt=${dt} + ods.cert_info.dt=${dt} + dim.dt=${pdt}
 
-INSERT OVERWRITE TABLE dim.dim_usr_user_ful_d PARTITION (dt='${dt}')
 WITH today_base AS (
     SELECT *
     FROM (
@@ -92,6 +91,7 @@ today_rebuilt AS (
     LEFT JOIN dim.dim_usr_user_ful_d o
         ON COALESCE(bu.id, ci.user_id) = o.user_id AND o.dt = '${pdt}'
 )
+INSERT OVERWRITE TABLE dim.dim_usr_user_ful_d PARTITION (dt='${dt}')
 SELECT
     user_id,
     appid,