in realbook/callbacks/debugging.py [0:0]
def monitoring_fn(self) -> None:
while not self.finished:
if self.should_dump_stacks and not self.dumped_stacks_already and self.mean_epoch_time:
sys.stderr.write(
f"Mean epoch time is {self.mean_epoch_time:2.2f} "
"seconds, but current epoch is taking "
f"{self.current_epoch_duration:2.2f} seconds. "
"Printing stacks for all threads:\n"
)
sys.stderr.flush()
faulthandler.dump_traceback()
sys.stderr.flush()
sys.stderr.write(
"If the current epoch does not finish within "
f"{5 * self.mean_epoch_time} seconds, this job "
"will exit.\n"
)
sys.stderr.flush()
self.dumped_stacks_already = True
if self.should_abort:
sys.stderr.write(
f"Mean epoch time is {self.mean_epoch_time:2.2f} "
"seconds, but current epoch is taking "
f"{self.current_epoch_duration:2.2f} seconds. "
"Aborting job.\n"
)
sys.stderr.flush()
self.abort()
time.sleep(self.CHECK_INTERVAL_SECONDS)