Coverage for klayout_pex/util/peak_memory.py: 64%

1# 

2# -------------------------------------------------------------------------------- 

3# SPDX-FileCopyrightText: 2024-2026 Martin Jan Köhler and Harald Pretl 

4# Johannes Kepler University, Institute for Integrated Circuits. 

5# 

6# This file is part of KPEX 

7# (see https://github.com/iic-jku/klayout-pex). 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21# SPDX-License-Identifier: GPL-3.0-or-later 

22# -------------------------------------------------------------------------------- 

23# 

24 

25""" 

26The peak memory of a run as resident set size, to size the next run on a larger layout (#409): 

27of this process, and of its largest subprocess that has ended (e.g. KLayout's LVS, FasterCap) 

28""" 

29 

30from __future__ import annotations 

31 

32from dataclasses import dataclass 

33import sys 

34from typing import * 

35 

36from ..log import info 

37 

38 

39@dataclass(frozen=True) 

40class PeakMemory: 

41 process: int # bytes 

42 

43 # bytes, 0 before one has ended, None where the platform doesn't tell (Windows), 

44 # NOTE: on Linux at least the resident set size of this process when it started it, which Linux carries over 

45 largest_subprocess: Optional[int] 

46 

47 

48def _windows_peak_working_set() -> int: 

49 """ 

50 :return: the peak working set of this process in bytes, the resident set size of Windows 

51 """ 

52 import ctypes 

53 from ctypes import wintypes 

54 

55 class ProcessMemoryCounters(ctypes.Structure): 

56 _fields_ = [('cb', wintypes.DWORD), 

57 ('PageFaultCount', wintypes.DWORD), 

58 ('PeakWorkingSetSize', ctypes.c_size_t), 

59 ('WorkingSetSize', ctypes.c_size_t), 

60 ('QuotaPeakPagedPoolUsage', ctypes.c_size_t), 

61 ('QuotaPagedPoolUsage', ctypes.c_size_t), 

62 ('QuotaPeakNonPagedPoolUsage', ctypes.c_size_t), 

63 ('QuotaNonPagedPoolUsage', ctypes.c_size_t), 

64 ('PagefileUsage', ctypes.c_size_t), 

65 ('PeakPagefileUsage', ctypes.c_size_t)] 

66 

67 # NOTE: a kernel32 of its own, as the types set on the functions of ctypes.windll.kernel32 would apply to all users 

68 kernel32 = ctypes.WinDLL('kernel32', use_last_error=True) 

69 kernel32.GetCurrentProcess.restype = wintypes.HANDLE 

70 get_process_memory_info = kernel32.K32GetProcessMemoryInfo 

71 get_process_memory_info.argtypes = [wintypes.HANDLE, ctypes.POINTER(ProcessMemoryCounters), wintypes.DWORD] 

72 get_process_memory_info.restype = wintypes.BOOL 

73 counters = ProcessMemoryCounters() 

74 counters.cb = ctypes.sizeof(counters) 

75 if not get_process_memory_info(kernel32.GetCurrentProcess(), ctypes.byref(counters), counters.cb): 

76 raise ctypes.WinError(ctypes.get_last_error()) 

77 return counters.PeakWorkingSetSize 

78 

79 

80def _linux_peak_resident_set_size() -> Optional[int]: 

81 """ 

82 :return: the peak resident set size of this process in bytes (VmHWM), None without /proc 

83 

84 NOTE: not ru_maxrss, which Linux carries over from what the process ran before exec, 

85 e.g. from the large process that started it (a test runner of 1.4 GiB) 

86 """ 

87 try: 

88 with open('/proc/self/status', encoding='utf-8') as f: 

89 sizes = [int(line.split()[1]) * 1024 for line in f if line.startswith('VmHWM:')] # NOTE: in kB 

90 except OSError: 

91 return None 

92 return sizes[0] if sizes else None 

93 

94 

95def peak_memory() -> PeakMemory: 

96 """ 

97 :return: the peak resident set sizes so far 

98 """ 

99 if sys.platform == 'win32': 

100 return PeakMemory(process=_windows_peak_working_set(), largest_subprocess=None) 

101 import resource 

102 unit = 1 if sys.platform == 'darwin' else 1024 # NOTE: ru_maxrss is in bytes on macOS, in KiB on Linux 

103 process = _linux_peak_resident_set_size() if sys.platform.startswith('linux') else None 

104 return PeakMemory(process=process or resource.getrusage(resource.RUSAGE_SELF).ru_maxrss * unit, 

105 largest_subprocess=resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss * unit) 

106 

107 

108def format_bytes(size: int) -> str: 

109 """ 

110 :return: e.g. '512 B', '245.3 MiB', '5.7 GiB' 

111 """ 

112 for unit, scale in (('GiB', 1 << 30), ('MiB', 1 << 20), ('KiB', 1 << 10)): 

113 if size >= scale: 

114 return f"{size / scale:.1f} {unit}" 

115 return f"{size} B" 

116 

117 

118def log_peak_memory(stage: str): 

119 """ 

120 Logs the peak memory so far, e.g. 'Peak memory (resident set size) after FasterCap: 245.3 MiB, 

121 largest subprocess 5.7 GiB', the subprocess only if it is the larger one, as on Linux a smaller one 

122 may be the size of this process when it started the subprocess (see PeakMemory) 

123 

124 :param stage: of the run, e.g. 'FasterCap' 

125 """ 

126 memory = peak_memory() 

127 subprocess = f", largest subprocess {format_bytes(memory.largest_subprocess)}" \ 

128 if memory.largest_subprocess and memory.largest_subprocess > memory.process else '' 

129 info(f"Peak memory (resident set size) after {stage}: {format_bytes(memory.process)}{subprocess}", 

130 stacklevel=3) # NOTE: logged as of the caller, the stage