|
|
@@ -17,6 +17,17 @@ SET hive.exec.dynamic.partition.mode=nonstrict;
|
|
|
SET hive.exec.max.dynamic.partitions=10000;
|
|
|
SET hive.exec.max.dynamic.partitions.pernode=10000;
|
|
|
|
|
|
+-- 资源放大(集群闲置,一次性重灌;L2 SET 覆盖 spark-tuning.conf 默认 15×2核/6g/shuffle200)。
|
|
|
+-- 动态分配是关的,杠杆是 executor.instances。按集群实际容量增减:单 executor 别超 YARN 容器上限
|
|
|
+-- (executor.memory+overhead ≤ yarn.scheduler.maximum-allocation-mb;cores ≤ maximum-allocation-vcores)。
|
|
|
+SET spark.executor.instances=20;
|
|
|
+SET spark.executor.cores=4;
|
|
|
+SET spark.executor.memory=8g;
|
|
|
+SET spark.executor.memoryOverhead=2g;
|
|
|
+SET spark.driver.memory=6g;
|
|
|
+SET spark.sql.shuffle.partitions=800;
|
|
|
+SET spark.default.parallelism=800;
|
|
|
+
|
|
|
INSERT OVERWRITE TABLE ods.ods_trd_card_group_order_info_inc_d PARTITION (dt)
|
|
|
SELECT
|
|
|
CAST(id AS BIGINT) AS id,
|