Parcourir la source

feat(prd): checklist 基础库切全量快照 ful_d(7天)+ful_m,删 inc

raw where 1=1 全量拉;ods ful_d 直 CAST 落 dt=T-1、前置 drop T-8 留 7 天;
ful_m 每月 1 号存上月月底、长期全留。无 seed/merge/dedup。

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
tianyu.chu il y a 1 semaine
Parent
commit
52e7892f82

+ 49 - 0
jobs/ods/prd/ods_prd_panini_checklist_base_info_ful_d.sql

@@ -0,0 +1,49 @@
+-- 作者:tianyu.chu
+-- 日期:2026-07-24
+-- 工单:(无)
+-- 目的:raw → ods 全量快照 —— 读 raw dt=${dt} 全量(一行一 id = 当前态),CAST 落 ods ful_d dt=${dt}
+-- 状态:[草案]
+-- 备注:sched=T,${dt}=T-1;维度全量快照模型(无 seed / 无 merge / 无 dedup,PG id 唯一)见 kb/24 §1.2.2;
+--       保留近 7 天:INSERT 前 DROP 掉 ${rdt}=T-8 一个分区(DS 传 rdt=$[yyyyMMdd-8],手动跑 -p rdt=<T-8>);
+--       静态分区写入 PARTITION(dt='${dt}'):源=raw、目标=ods,不同表无读写冲突。
+
+ALTER TABLE ods.ods_prd_panini_checklist_base_info_ful_d DROP IF EXISTS PARTITION (dt='${rdt}');
+
+INSERT OVERWRITE TABLE ods.ods_prd_panini_checklist_base_info_ful_d PARTITION (dt='${dt}')
+SELECT
+    CAST(id                  AS BIGINT)        AS id,
+    code                                       AS code,
+    year                                       AS year,
+    sport                                      AS sport,
+    manufacturer                               AS manufacturer,
+    sets                                       AS sets,
+    sets_version                               AS sets_version,
+    display_name                               AS display_name,
+    type                                       AS type,
+    CAST(num                 AS BIGINT)        AS num,
+    CAST(use_num             AS BIGINT)        AS use_num,
+    CAST(status              AS BIGINT)        AS status,
+    remark                                     AS remark,
+    CAST(lot                 AS BIGINT)        AS lot,
+    CAST(del_flg             AS BIGINT)        AS del_flg,
+    carmichael_img_type                        AS carmichael_img_type,
+    CAST(create_time         AS TIMESTAMP)     AS create_time,
+    create_by                                  AS create_by,
+    CAST(update_time         AS TIMESTAMP)     AS update_time,
+    update_by                                  AS update_by,
+    import_type                                AS import_type,
+    CAST(report_flag         AS BIGINT)        AS report_flag,
+    base_config                                AS base_config,
+    CAST(base_price          AS DECIMAL(20,4)) AS base_price,
+    CAST(min_reference_price AS DECIMAL(20,4)) AS min_reference_price,
+    CAST(max_reference_price AS DECIMAL(20,4)) AS max_reference_price,
+    prop1                                      AS prop1,
+    prop2                                      AS prop2,
+    prop3                                      AS prop3,
+    prop4                                      AS prop4,
+    data_version                               AS data_version,
+    first_sport                                AS first_sport,
+    sets_display_name                          AS sets_display_name,
+    CASE WHEN del_flg = '1' THEN TRUE ELSE FALSE END AS is_deleted
+FROM raw.raw_prd_panini_checklist_base_info_ful_d
+WHERE dt = '${dt}';

+ 17 - 0
jobs/ods/prd/ods_prd_panini_checklist_base_info_ful_m.sql

@@ -0,0 +1,17 @@
+-- 作者:tianyu.chu
+-- 日期:2026-07-24
+-- 工单:(无)
+-- 目的:月快照 —— 每月 1 号读 ods ful_d 的 T-1 分区(恰=上月月底),另存进长留月表 ful_m dt=${dt}
+-- 状态:[草案]
+-- 备注:sched=T=每月 1 号,${dt}=T-1=上月月底(如 8/1 跑 → ${dt}=20260731);模型见 kb/24 §1.2.1;
+--       长期全留,无 retention drop;源=ods ful_d 同 schema,直拷;静态分区 PARTITION(dt='${dt}')。
+
+INSERT OVERWRITE TABLE ods.ods_prd_panini_checklist_base_info_ful_m PARTITION (dt='${dt}')
+SELECT
+    id, code, year, sport, manufacturer, sets, sets_version, display_name, type,
+    num, use_num, status, remark, lot, del_flg, carmichael_img_type,
+    create_time, create_by, update_time, update_by, import_type, report_flag,
+    base_config, base_price, min_reference_price, max_reference_price,
+    prop1, prop2, prop3, prop4, data_version, first_sport, sets_display_name, is_deleted
+FROM ods.ods_prd_panini_checklist_base_info_ful_d
+WHERE dt = '${dt}';

+ 0 - 56
jobs/ods/prd/ods_prd_panini_checklist_base_info_inc_d.sql

@@ -1,56 +0,0 @@
--- 作者:tianyu.chu
--- 日期:2026-05-07
--- 工单:(无)
--- 目的:raw → ods 增量同步(ADR-03);双源 union (raw dt=${dt} + raw dt=${pdt}) + DATE_FORMAT(COALESCE(update_time, create_time))=${dt} 过滤 + (id, ods_dt) dedupe + 动态分区写入
--- 状态:[草案]
--- 备注:sched=T,${dt}=业务日 T-1(yyyyMMdd),${pdt}=T-2
-
-INSERT OVERWRITE TABLE ods.ods_prd_panini_checklist_base_info_inc_d PARTITION (dt)
-SELECT
-    CAST(id                 AS BIGINT)        AS id,
-    code                                      AS code,
-    year                                      AS year,
-    sport                                     AS sport,
-    manufacturer                              AS manufacturer,
-    sets                                      AS sets,
-    sets_version                              AS sets_version,
-    display_name                              AS display_name,
-    type                                      AS type,
-    CAST(num                AS BIGINT)        AS num,
-    CAST(use_num            AS BIGINT)        AS use_num,
-    CAST(status             AS BIGINT)        AS status,
-    remark                                    AS remark,
-    CAST(lot                AS BIGINT)        AS lot,
-    CAST(del_flg            AS BIGINT)        AS del_flg,
-    carmichael_img_type                       AS carmichael_img_type,
-    CAST(create_time        AS TIMESTAMP)     AS create_time,
-    create_by                                 AS create_by,
-    CAST(update_time        AS TIMESTAMP)     AS update_time,
-    update_by                                 AS update_by,
-    import_type                               AS import_type,
-    CAST(report_flag        AS BIGINT)        AS report_flag,
-    base_config                               AS base_config,
-    CAST(base_price         AS DECIMAL(20,4)) AS base_price,
-    CAST(min_reference_price AS DECIMAL(20,4)) AS min_reference_price,
-    CAST(max_reference_price AS DECIMAL(20,4)) AS max_reference_price,
-    prop1                                     AS prop1,
-    prop2                                     AS prop2,
-    prop3                                     AS prop3,
-    prop4                                     AS prop4,
-    data_version                              AS data_version,
-    first_sport                               AS first_sport,
-    sets_display_name                         AS sets_display_name,
-    CASE WHEN del_flg = '1' THEN TRUE ELSE FALSE END AS is_deleted,
-    ods_dt                                    AS dt
-FROM (
-    SELECT *,
-        DATE_FORMAT(COALESCE(NULLIF(update_time, ''), create_time), 'yyyyMMdd') AS ods_dt,
-        ROW_NUMBER() OVER (
-            PARTITION BY id, DATE_FORMAT(COALESCE(NULLIF(update_time, ''), create_time), 'yyyyMMdd')
-            ORDER BY COALESCE(NULLIF(update_time, ''), create_time) DESC
-        ) AS rn
-    FROM raw.raw_prd_panini_checklist_base_info_inc_d
-    WHERE dt IN ('${dt}', '${pdt}')
-      AND DATE_FORMAT(COALESCE(NULLIF(update_time, ''), create_time), 'yyyyMMdd') = '${dt}'
-) t
-WHERE t.rn = 1;

+ 9 - 9
jobs/raw/prd/raw_prd_panini_checklist_base_info_inc_d.ini → jobs/raw/prd/raw_prd_panini_checklist_base_info_ful_d.ini

@@ -1,12 +1,12 @@
 ; 作者:tianyu.chu
 ; 作者:tianyu.chu
-; 日期:2026-04-30
+; 日期:2026-07-24
 ; 工单:(无)
 ; 工单:(无)
-; 目的:PG public.panini_checklist_base_info -> Hive raw.raw_prd_panini_checklist_base_info_inc_d
-;       基础库 checklist(panini_checklist_base_info),日常增量同步,时间锚点 = update_time
+; 目的:PG public.panini_checklist_base_info -> Hive raw.raw_prd_panini_checklist_base_info_ful_d
+;       基础库 checklist,维度全量快照日拉(where 1=1,非增量窗),写 dt=T-1;模型见 kb/24 §1.2.2
 ;
 ;
-; 配套 DDL:manual/ddl/raw/prd/raw_prd_panini_checklist_base_info_inc_d_create.sql
-; 配套 mask 配置:jobs/raw/prd/raw_prd_panini_checklist_base_info_inc_d.mask.ini
-; 状态:第一版(未与业务库后端对账)
+; 配套 DDL:manual/ddl/raw/prd/raw_prd_panini_checklist_base_info_ful_d_create.sql
+; 配套 mask 配置:jobs/raw/prd/raw_prd_panini_checklist_base_info_ful_d.mask.ini
+; 状态:第一版
 
 
 [reader]
 [reader]
 dataSource = postgresql/prd-poyee-aliyun
 dataSource = postgresql/prd-poyee-aliyun
@@ -14,18 +14,18 @@ database = hobby_stocks
 table = public.panini_checklist_base_info
 table = public.panini_checklist_base_info
 column = id,code,year,sport,manufacturer,sets,sets_version,display_name,type,num,use_num,status,remark,lot,del_flg,carmichael_img_type,create_time,create_by,update_time,update_by,import_type,report_flag,base_config,base_price,min_reference_price,max_reference_price,prop1,prop2,prop3,prop4,data_version,first_sport,sets_display_name
 column = id,code,year,sport,manufacturer,sets,sets_version,display_name,type,num,use_num,status,remark,lot,del_flg,carmichael_img_type,create_time,create_by,update_time,update_by,import_type,report_flag,base_config,base_price,min_reference_price,max_reference_price,prop1,prop2,prop3,prop4,data_version,first_sport,sets_display_name
 columnType =
 columnType =
-where = COALESCE(update_time, create_time) >= '${start_date}' AND COALESCE(update_time, create_time) < '${stop_date}'
+where = 1=1
 querySql =
 querySql =
 splitPk = id
 splitPk = id
 fetchSize = 1000
 fetchSize = 1000
 
 
 [writer]
 [writer]
 dataSource = hdfs/prd-hdfs-ha
 dataSource = hdfs/prd-hdfs-ha
-path = /user/hive/warehouse/raw.db/raw_prd_panini_checklist_base_info_inc_d/dt=${dt}/
+path = /user/hive/warehouse/raw.db/raw_prd_panini_checklist_base_info_ful_d/dt=${dt}/
 column = id,code,year,sport,manufacturer,sets,sets_version,display_name,type,num,use_num,status,remark,lot,del_flg,carmichael_img_type,create_time,create_by,update_time,update_by,import_type,report_flag,base_config,base_price,min_reference_price,max_reference_price,prop1,prop2,prop3,prop4,data_version,first_sport,sets_display_name
 column = id,code,year,sport,manufacturer,sets,sets_version,display_name,type,num,use_num,status,remark,lot,del_flg,carmichael_img_type,create_time,create_by,update_time,update_by,import_type,report_flag,base_config,base_price,min_reference_price,max_reference_price,prop1,prop2,prop3,prop4,data_version,first_sport,sets_display_name
 columnType =
 columnType =
 fileType = orc
 fileType = orc
-fileName = raw_prd_panini_checklist_base_info_inc_d
+fileName = raw_prd_panini_checklist_base_info_ful_d
 encoding = UTF-8
 encoding = UTF-8
 writeMode = truncate
 writeMode = truncate
 fieldDelimiter = \t
 fieldDelimiter = \t

+ 4 - 4
jobs/raw/prd/raw_prd_panini_checklist_base_info_inc_d.mask.ini → jobs/raw/prd/raw_prd_panini_checklist_base_info_ful_d.mask.ini

@@ -1,6 +1,6 @@
-; 表名:raw_prd_panini_checklist_base_info_inc_d
-; 配套 sync ini:jobs/raw/prd/raw_prd_panini_checklist_base_info_inc_d.ini
-; 配套 DDL:manual/ddl/raw/prd/raw_prd_panini_checklist_base_info_inc_d_create.sql
+; 表名:raw_prd_panini_checklist_base_info_ful_d
+; 配套 sync ini:jobs/raw/prd/raw_prd_panini_checklist_base_info_ful_d.ini
+; 配套 DDL:manual/ddl/raw/prd/raw_prd_panini_checklist_base_info_ful_d_create.sql
 ; 配套建模文档:kb/22-业务库raw建模.md §6
 ; 配套建模文档:kb/22-业务库raw建模.md §6
 ;
 ;
 ; 字段决策来源:inbox 3.2.2.5 商品域建模.md(分析师库 ods 草稿) vs PG 业务库实际 schema 对账。
 ; 字段决策来源:inbox 3.2.2.5 商品域建模.md(分析师库 ods 草稿) vs PG 业务库实际 schema 对账。
@@ -8,7 +8,7 @@
 ;     method 映射:y → trim、y(hash) → md5、y(sync -month) → month_trunc。
 ;     method 映射:y → trim、y(hash) → md5、y(sync -month) → month_trunc。
 ;   - 隐性裁:PG 业务库有 / inbox 草稿无的字段(上游变更未通知下游,待与业务库后端对账)。
 ;   - 隐性裁:PG 业务库有 / inbox 草稿无的字段(上游变更未通知下游,待与业务库后端对账)。
 ;
 ;
-; 2026-04-30 第一版:本表 PG 字段集与 inbox 完全一致,无明性裁、无隐性裁,全字段入仓。
+; 本表 PG 字段集与 inbox 完全一致,无明性裁、无隐性裁,全字段入仓。
 ; 保留空 [mask] 段以满足 sync 生成器加载约定。
 ; 保留空 [mask] 段以满足 sync 生成器加载约定。
 
 
 [mask]
 [mask]

+ 9 - 9
manual/ddl/ods/prd/ods_prd_panini_checklist_base_info_inc_d_create.sql → manual/ddl/ods/prd/ods_prd_panini_checklist_base_info_ful_d_create.sql

@@ -1,13 +1,13 @@
 -- 作者:tianyu.chu
 -- 作者:tianyu.chu
--- 日期:2026-05-06
--- 工单:<TODO>
--- 目的:<TODO>
--- 状态:[执行]
--- 备注:<TODO>
+-- 日期:2026-07-24
+-- 工单:(无)
+-- 目的:ods 层维度全量快照表(typed),承接 raw ful_d 每日全量;按 dt 分区 ORC,保留近 7 天
+-- 状态:[执行]
+-- 备注:配套 SQL = jobs/ods/prd/ods_prd_panini_checklist_base_info_ful_d.sql;模型见 kb/24 §1.2.2
 
 
-DROP TABLE IF EXISTS ods.ods_prd_panini_checklist_base_info_inc_d;
+DROP TABLE IF EXISTS ods.ods_prd_panini_checklist_base_info_ful_d;
 
 
-CREATE EXTERNAL TABLE IF NOT EXISTS ods.ods_prd_panini_checklist_base_info_inc_d (
+CREATE EXTERNAL TABLE IF NOT EXISTS ods.ods_prd_panini_checklist_base_info_ful_d (
     id                     BIGINT         COMMENT '主键',
     id                     BIGINT         COMMENT '主键',
     code                   STRING         COMMENT '编码',
     code                   STRING         COMMENT '编码',
     year                   STRING         COMMENT '年份(赛季)',
     year                   STRING         COMMENT '年份(赛季)',
@@ -43,7 +43,7 @@ CREATE EXTERNAL TABLE IF NOT EXISTS ods.ods_prd_panini_checklist_base_info_inc_d
     sets_display_name      STRING         COMMENT '系列简称',
     sets_display_name      STRING         COMMENT '系列简称',
     is_deleted             BOOLEAN        COMMENT '软删除归一(CASE WHEN del_* THEN TRUE)'
     is_deleted             BOOLEAN        COMMENT '软删除归一(CASE WHEN del_* THEN TRUE)'
 )
 )
-COMMENT '基础库 checklist'
+COMMENT '基础库 checklist(全量快照,保留近 7 天)'
 PARTITIONED BY (dt STRING)
 PARTITIONED BY (dt STRING)
 STORED AS ORC
 STORED AS ORC
-LOCATION '/user/hive/warehouse/ods.db/ods_prd_panini_checklist_base_info_inc_d';
+LOCATION '/user/hive/warehouse/ods.db/ods_prd_panini_checklist_base_info_ful_d';

+ 50 - 0
manual/ddl/ods/prd/ods_prd_panini_checklist_base_info_ful_m_create.sql

@@ -0,0 +1,50 @@
+-- 作者:tianyu.chu
+-- 日期:2026-07-24
+-- 工单:(无)
+-- 目的:ods 层维度月快照表(typed),每月 1 号存上月月底全量;按 dt 分区 ORC,长期全留
+-- 状态:[待执行]
+-- 备注:配套 SQL = jobs/ods/prd/ods_prd_panini_checklist_base_info_ful_m.sql;模型见 kb/24 §1.2.1;
+--       schema 与 ods ful_d 完全一致(月快照是 ful_d 月底那版的长留副本)
+
+DROP TABLE IF EXISTS ods.ods_prd_panini_checklist_base_info_ful_m;
+
+CREATE EXTERNAL TABLE IF NOT EXISTS ods.ods_prd_panini_checklist_base_info_ful_m (
+    id                     BIGINT         COMMENT '主键',
+    code                   STRING         COMMENT '编码',
+    year                   STRING         COMMENT '年份(赛季)',
+    sport                  STRING         COMMENT '运动类型',
+    manufacturer           STRING         COMMENT '厂商(发行商)',
+    sets                   STRING         COMMENT '系列',
+    sets_version           STRING         COMMENT '版本',
+    display_name           STRING         COMMENT '别名(显示名)',
+    type                   STRING         COMMENT '可组类型',
+    num                    BIGINT         COMMENT 'list数量',
+    use_num                BIGINT         COMMENT '使用次数',
+    status                 BIGINT         COMMENT '状态',
+    remark                 STRING         COMMENT '描述',
+    lot                    BIGINT         COMMENT '是否是lot',
+    del_flg                BIGINT         COMMENT '是否删除:0=正常,1=删除',
+    carmichael_img_type    STRING         COMMENT '卡密图片适用类型:球员,球队,other,all',
+    create_time            TIMESTAMP      COMMENT '',
+    create_by              STRING         COMMENT '',
+    update_time            TIMESTAMP      COMMENT '',
+    update_by              STRING         COMMENT '',
+    import_type            STRING         COMMENT '导入类型',
+    report_flag            BIGINT         COMMENT '是否强制使用新版报告形式,默认0:非,1:强制',
+    base_config            STRING         COMMENT '配置',
+    base_price             DECIMAL(20,4)  COMMENT '指导价: 元/箱',
+    min_reference_price    DECIMAL(20,4)  COMMENT '参考价范围-最小',
+    max_reference_price    DECIMAL(20,4)  COMMENT '参考价范围-最大',
+    prop1                  STRING         COMMENT '',
+    prop2                  STRING         COMMENT '',
+    prop3                  STRING         COMMENT '',
+    prop4                  STRING         COMMENT '',
+    data_version           STRING         COMMENT '数据版本',
+    first_sport            STRING         COMMENT '大分类-品类(运动类型)',
+    sets_display_name      STRING         COMMENT '系列简称',
+    is_deleted             BOOLEAN        COMMENT '软删除归一(CASE WHEN del_* THEN TRUE)'
+)
+COMMENT '基础库 checklist(月快照,每月 1 号存上月月底,长期全留)'
+PARTITIONED BY (dt STRING)
+STORED AS ORC
+LOCATION '/user/hive/warehouse/ods.db/ods_prd_panini_checklist_base_info_ful_m';

+ 10 - 10
manual/ddl/raw/prd/raw_prd_panini_checklist_base_info_inc_d_create.sql → manual/ddl/raw/prd/raw_prd_panini_checklist_base_info_ful_d_create.sql

@@ -1,15 +1,15 @@
 -- 作者:tianyu.chu
 -- 作者:tianyu.chu
--- 日期:2026-04-30
+-- 日期:2026-07-24
 -- 工单:(无)
 -- 工单:(无)
--- 目的:raw 层贴源增量表,承接 PG public.panini_checklist_base_info 日常增量;全字段 STRING,按 dt 分区 ORC 存储
--- 状态:[执行]
--- 备注:配套 sync ini = jobs/raw/prd/raw_prd_panini_checklist_base_info_inc_d.ini;
---       配套 mask 配置 = jobs/raw/prd/raw_prd_panini_checklist_base_info_inc_d.mask.ini;
---       字段决策溯源 = kb/22-业务库raw建模.md
+-- 目的:raw 层贴源全量快照表,承接 PG public.panini_checklist_base_info 每日全量;全字段 STRING,按 dt 分区 ORC 存储
+-- 状态:[执行]
+-- 备注:配套 sync ini = jobs/raw/prd/raw_prd_panini_checklist_base_info_ful_d.ini;
+--       配套 mask 配置 = jobs/raw/prd/raw_prd_panini_checklist_base_info_ful_d.mask.ini;
+--       维度全量快照模型见 kb/24 §1.2.2;字段决策溯源 = kb/22-业务库raw建模.md §6
 
 
-DROP TABLE IF EXISTS raw.raw_prd_panini_checklist_base_info_inc_d;
+DROP TABLE IF EXISTS raw.raw_prd_panini_checklist_base_info_ful_d;
 
 
-CREATE EXTERNAL TABLE IF NOT EXISTS raw.raw_prd_panini_checklist_base_info_inc_d (
+CREATE EXTERNAL TABLE IF NOT EXISTS raw.raw_prd_panini_checklist_base_info_ful_d (
     id                     STRING COMMENT '主键',
     id                     STRING COMMENT '主键',
     code                   STRING COMMENT '编码',
     code                   STRING COMMENT '编码',
     year                   STRING COMMENT '年份(赛季)',
     year                   STRING COMMENT '年份(赛季)',
@@ -44,7 +44,7 @@ CREATE EXTERNAL TABLE IF NOT EXISTS raw.raw_prd_panini_checklist_base_info_inc_d
     first_sport            STRING COMMENT '大分类-品类(运动类型)',
     first_sport            STRING COMMENT '大分类-品类(运动类型)',
     sets_display_name      STRING COMMENT '系列简称'
     sets_display_name      STRING COMMENT '系列简称'
 )
 )
-COMMENT '基础库 checklist'
+COMMENT '基础库 checklist(全量快照)'
 PARTITIONED BY (dt STRING)
 PARTITIONED BY (dt STRING)
 STORED AS ORC
 STORED AS ORC
-LOCATION '/user/hive/warehouse/raw.db/raw_prd_panini_checklist_base_info_inc_d';
+LOCATION '/user/hive/warehouse/raw.db/raw_prd_panini_checklist_base_info_ful_d';