|
@@ -60,7 +60,8 @@ def main():
|
|
|
rc_raw = run(['python3', STARTER, '-f', RAW_SQL, '-u', UDF,
|
|
rc_raw = run(['python3', STARTER, '-f', RAW_SQL, '-u', UDF,
|
|
|
'-sc', 'spark.executor.memoryOverhead=4g'], cwd=PROJECT_ROOT)
|
|
'-sc', 'spark.executor.memoryOverhead=4g'], cwd=PROJECT_ROOT)
|
|
|
rc_ods = run(['python3', STARTER, '-f', ODS_SQL,
|
|
rc_ods = run(['python3', STARTER, '-f', ODS_SQL,
|
|
|
- '-p', 'start_date=%s' % start, '-p', 'stop_date=%s' % stop], cwd=PROJECT_ROOT)
|
|
|
|
|
|
|
+ '-p', 'start_date=%s' % start, '-p', 'stop_date=%s' % stop,
|
|
|
|
|
+ '-sc', 'spark.executor.memoryOverhead=4g'], cwd=PROJECT_ROOT)
|
|
|
run(['hdfs', 'dfs', '-rm', '-r', '-f', '-skipTrash', STAGING])
|
|
run(['hdfs', 'dfs', '-rm', '-r', '-f', '-skipTrash', STAGING])
|
|
|
|
|
|
|
|
if rc_raw or rc_ods:
|
|
if rc_raw or rc_ods:
|