|
@@ -132,9 +132,7 @@ class DataxImport(_BaseDatax):
|
|
|
parallel: bool = False,
|
|
parallel: bool = False,
|
|
|
skip_partition: bool = False,
|
|
skip_partition: bool = False,
|
|
|
skip_datax: bool = False,
|
|
skip_datax: bool = False,
|
|
|
- skip_check: bool = False,
|
|
|
|
|
- speed_overrides: Optional[dict] = None,
|
|
|
|
|
- extra_partition_tables: Optional[List[str]] = None) -> int:
|
|
|
|
|
|
|
+ speed_overrides: Optional[dict] = None) -> int:
|
|
|
"""
|
|
"""
|
|
|
Returns: 失败任务数(0 = 全部成功)
|
|
Returns: 失败任务数(0 = 全部成功)
|
|
|
"""
|
|
"""
|
|
@@ -150,15 +148,11 @@ class DataxImport(_BaseDatax):
|
|
|
ddl = partition.parse_ini_partition(ini, stop_date)
|
|
ddl = partition.parse_ini_partition(ini, stop_date)
|
|
|
if ddl:
|
|
if ddl:
|
|
|
ddls.append(ddl)
|
|
ddls.append(ddl)
|
|
|
- if extra_partition_tables:
|
|
|
|
|
- dt = partition.compute_partition_dt(stop_date)
|
|
|
|
|
- for tbl in extra_partition_tables:
|
|
|
|
|
- ddls.append('ALTER TABLE {tbl} ADD IF NOT EXISTS PARTITION(dt={dt});'.format(
|
|
|
|
|
- tbl=tbl, dt=dt))
|
|
|
|
|
partition.execute_ddls(ddls)
|
|
partition.execute_ddls(ddls)
|
|
|
|
|
|
|
|
- run_one = self._make_run_one(start_date, stop_date, host, use_random, parallel, skip_datax, skip_check,
|
|
|
|
|
- speed_overrides)
|
|
|
|
|
|
|
+ # import 的 reader 永非 hdfs,runner._hdfs_src_check 早返 n/a,skip_check 透传无意义,固定 False
|
|
|
|
|
+ run_one = self._make_run_one(start_date, stop_date, host, use_random, parallel, skip_datax,
|
|
|
|
|
+ skip_check=False, speed_overrides=speed_overrides)
|
|
|
_success, failed = batch.run_batch(ini_list, run_one, parallel=parallel)
|
|
_success, failed = batch.run_batch(ini_list, run_one, parallel=parallel)
|
|
|
return failed
|
|
return failed
|
|
|
|
|
|
|
@@ -172,9 +166,7 @@ class DataxImport(_BaseDatax):
|
|
|
parallel: bool = False,
|
|
parallel: bool = False,
|
|
|
skip_partition: bool = False,
|
|
skip_partition: bool = False,
|
|
|
skip_datax: bool = False,
|
|
skip_datax: bool = False,
|
|
|
- skip_check: bool = False,
|
|
|
|
|
- speed_overrides: Optional[dict] = None,
|
|
|
|
|
- extra_partition_tables: Optional[List[str]] = None) -> int:
|
|
|
|
|
|
|
+ speed_overrides: Optional[dict] = None) -> int:
|
|
|
"""
|
|
"""
|
|
|
存量回填:start_date/stop_date 作外层范围 [含, 不含),按日循环调 self.run() 单日语义。
|
|
存量回填:start_date/stop_date 作外层范围 [含, 不含),按日循环调 self.run() 单日语义。
|
|
|
|
|
|
|
@@ -203,9 +195,7 @@ class DataxImport(_BaseDatax):
|
|
|
parallel=parallel,
|
|
parallel=parallel,
|
|
|
skip_partition=skip_partition,
|
|
skip_partition=skip_partition,
|
|
|
skip_datax=skip_datax,
|
|
skip_datax=skip_datax,
|
|
|
- skip_check=skip_check,
|
|
|
|
|
speed_overrides=speed_overrides,
|
|
speed_overrides=speed_overrides,
|
|
|
- extra_partition_tables=extra_partition_tables,
|
|
|
|
|
)
|
|
)
|
|
|
if failed > 0:
|
|
if failed > 0:
|
|
|
total_failed += failed
|
|
total_failed += failed
|