ods_trd_card_group_info_ful_d.sql 3.5 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455
  1. -- 作者:tianyu.chu
  2. -- 日期:2026-07-17
  3. -- 工单:(无)
  4. -- 目的:拼团全量最新态日常 merge —— full dt=${dt} ← full dt=${pdt} ⨝ inc dt=${dt},按 id 取最新一版
  5. -- 状态:[草案]
  6. -- 备注:sched=T,${dt}=业务日 T-1、${pdt}=T-2(即前一天的 full)。
  7. -- **静态分区写入**(PARTITION(dt='${dt}'))是硬要求:本 SQL 读 full dt=${pdt} 又写 full,
  8. -- 若用动态分区(PARTITION(dt)),输出路径=表根目录、与读的分区路径重叠,Spark 会报
  9. -- "Cannot overwrite a path that is also being read from"。静态分区的输出是具体分区目录,不重叠。
  10. -- 起点由 manual/backfill/20260717_..._ful_d_seed.sql 一次性 seed;断链需重新 seed。
  11. -- 历史多版本在 inc 表(拉链底座),本表只保当前态。lock 是保留字,SELECT 里必须带反引号。
  12. -- **分区保留 2 天**(只留 ${dt} 与 ${pdt}):文末 ALTER 精确 drop ${rdt}(=${dt}-2,即 T-3),
  13. -- 每天正好掉一个。用精确分区值而非 dt < 'x'——Spark SQL 的 DROP PARTITION 只接受 col=value,
  14. -- 比较式是 Hive 语法、Spark 解析不了。DS 需额外传参 rdt=$[yyyyMMdd-3];手动跑加 -p rdt=<T-3>。
  15. -- 代价:漏跑一天则那天本该掉的分区不会被补 drop,需手工清;断链重新 seed 即可。
  16. INSERT OVERWRITE TABLE ods.ods_trd_card_group_info_ful_d PARTITION (dt='${dt}')
  17. SELECT
  18. id, merchant_id, appid, name, code,
  19. status, specs, type, random_type, total_price,
  20. copies, unit_price, sold_copies, release_time, cycle,
  21. show_applet, title, msg, remark, create_time,
  22. update_by, update_time, order_quota_min, order_quota_max, user_quota_max,
  23. start_time, marketing_info, reviewmsg, `lock`, commission_rate,
  24. year, sport, manufacturer, sets, act,
  25. config, info_config, total_num, banner_end_time, add_banner,
  26. finished_time, display_name, group_sets_no, close_payment_time, confirm_send_time,
  27. close_payment_status, open_card, close_payment_record, group_full_time, live_create_time,
  28. live_start_time, live_end_time, report_start_time, report_end_time, report_review_num,
  29. report_review_first_time, report_review_end_time, review_hold_time, review_approval_time, review_num,
  30. config_json, free_flag, mer_name, change_type, act_price,
  31. act_config_json, real_sold_num, weight, hot_type, team_first,
  32. prop1, prop2, prop3, point_rate, point_max,
  33. point_min, list_id, list_code, mix_copies, sub_type,
  34. act_point_type, payment_method, payment_total_price, payment_commission, payment_finished_price,
  35. payment_remain_price, payment_online_price, exclusive, has_bg, merchant_sort,
  36. del_flg, del_time, review_account, act_id, sold_end_time,
  37. panini_list_id, hot_type_config, goods_type, report_flag, use_coupon,
  38. user_level, custom, gift_card_id, group_show_name, min_card_num,
  39. act_type, waring_type, compensation_status, point_type, first_act_config,
  40. gift_config, version, extra_prop, use_member_discount, merchant_open,
  41. is_deleted
  42. FROM (
  43. SELECT *,
  44. ROW_NUMBER() OVER (PARTITION BY id ORDER BY COALESCE(update_time, create_time) DESC) AS rn
  45. FROM (
  46. SELECT * FROM ods.ods_trd_card_group_info_ful_d WHERE dt = '${pdt}'
  47. UNION ALL
  48. SELECT * FROM ods.ods_trd_card_group_info_inc_d WHERE dt = '${dt}'
  49. ) u
  50. ) t
  51. WHERE t.rn = 1;
  52. -- 保留 2 天:drop 掉 ${dt}-2(T-3)这一个分区
  53. ALTER TABLE ods.ods_trd_card_group_info_ful_d DROP IF EXISTS PARTITION (dt='${rdt}');