diff --git a/debug/accuracy_tools/grad_tool/grad_ms/global_context.py b/debug/accuracy_tools/grad_tool/grad_ms/global_context.py index b988d3503399f6ffde5646c4a1ae4d64ad4ffa11..9b9ff2e56df96e2184471bbef483257b3cb70305 100644 --- a/debug/accuracy_tools/grad_tool/grad_ms/global_context.py +++ b/debug/accuracy_tools/grad_tool/grad_ms/global_context.py @@ -69,9 +69,10 @@ class GlobalContext: else: type_str = "string" if value and isinstance(value, list): - if not isinstance(value[0], dtype): - print_warn_log(f"Invalid {name} which must be None or list of {type_str}") - value = None + for val in value: + if not isinstance(val, dtype): + print_warn_log(f"Invalid {name} which must be None or list of {type_str}") + return self._setting[name] = value diff --git a/debug/accuracy_tools/grad_tool/grad_ms/grad_analyzer.py b/debug/accuracy_tools/grad_tool/grad_ms/grad_analyzer.py index 1b11ce63c882f364727b503a9db1af713cc4a88e..d710b8eeb3bcb7874b29fd10d37033d84c34de55 100644 --- a/debug/accuracy_tools/grad_tool/grad_ms/grad_analyzer.py +++ b/debug/accuracy_tools/grad_tool/grad_ms/grad_analyzer.py @@ -97,6 +97,8 @@ class CSVGenerator(Process): while True: if not os.path.exists(self.dump_dir): time.sleep(0.1) + if self.stop_event.is_set(): + break continue npy_files = os.listdir(self.dump_dir) npy_files.sort(key=lambda x: int(x.split("_")[0])) @@ -104,7 +106,8 @@ class CSVGenerator(Process): empty = len(os.listdir(self.dump_dir)) == 0 if self.stop_event.is_set() and empty and self.last_finish: break - shutil.rmtree(self.dump_dir) + if os.path.exists(self.dump_dir): + shutil.rmtree(self.dump_dir) def stop(self): self.stop_event.set()