|
@@ -17,16 +17,16 @@ SET hive.exec.dynamic.partition.mode=nonstrict;
|
|
|
SET hive.exec.max.dynamic.partitions=10000;
|
|
SET hive.exec.max.dynamic.partitions=10000;
|
|
|
SET hive.exec.max.dynamic.partitions.pernode=10000;
|
|
SET hive.exec.max.dynamic.partitions.pernode=10000;
|
|
|
|
|
|
|
|
--- 资源放大(集群闲置,一次性重灌;L2 SET 覆盖 spark-tuning.conf 默认 15×2核/6g/shuffle200)。
|
|
|
|
|
--- 动态分配是关的,杠杆是 executor.instances。按集群实际容量增减:单 executor 别超 YARN 容器上限
|
|
|
|
|
--- (executor.memory+overhead ≤ yarn.scheduler.maximum-allocation-mb;cores ≤ maximum-allocation-vcores)。
|
|
|
|
|
-SET spark.executor.instances=20;
|
|
|
|
|
|
|
+-- 资源放大(一次性重灌;L2 SET 覆盖 spark-tuning.conf 默认 15×2核/6g/shuffle200)。动态分配关,杠杆是 instances。
|
|
|
|
|
+-- 实测集群:3 节点 × 48 vcore / 150GB = 144 vcore / 450GB,全闲。
|
|
|
|
|
+-- 配 30 executor × 4核 = 120 核(83%),每 executor 8g+2g=10g → 300GB(67%),每节点 10 个(40核/100GB,留头寸)。
|
|
|
|
|
+SET spark.executor.instances=30;
|
|
|
SET spark.executor.cores=4;
|
|
SET spark.executor.cores=4;
|
|
|
SET spark.executor.memory=8g;
|
|
SET spark.executor.memory=8g;
|
|
|
SET spark.executor.memoryOverhead=2g;
|
|
SET spark.executor.memoryOverhead=2g;
|
|
|
-SET spark.driver.memory=6g;
|
|
|
|
|
-SET spark.sql.shuffle.partitions=800;
|
|
|
|
|
-SET spark.default.parallelism=800;
|
|
|
|
|
|
|
+SET spark.driver.memory=8g;
|
|
|
|
|
+SET spark.sql.shuffle.partitions=1000;
|
|
|
|
|
+SET spark.default.parallelism=1000;
|
|
|
|
|
|
|
|
INSERT OVERWRITE TABLE ods.ods_trd_card_group_order_info_inc_d PARTITION (dt)
|
|
INSERT OVERWRITE TABLE ods.ods_trd_card_group_order_info_inc_d PARTITION (dt)
|
|
|
SELECT
|
|
SELECT
|