1# 2# Copyright OpenEmbedded Contributors 3# 4# SPDX-License-Identifier: GPL-2.0-only 5# 6# Implements system state sampling. Called by buildstats.bbclass. 7# Because it is a real Python module, it can hold persistent state, 8# like open log files and the time of the last sampling. 9 10import time 11import re 12import bb.event 13 14class SystemStats: 15 def __init__(self, d): 16 bn = d.getVar('BUILDNAME') 17 bsdir = os.path.join(d.getVar('BUILDSTATS_BASE'), bn) 18 bb.utils.mkdirhier(bsdir) 19 file_handlers = [('diskstats', self._reduce_diskstats), 20 ('meminfo', self._reduce_meminfo), 21 ('stat', self._reduce_stat)] 22 23 # Some hosts like openSUSE have readable /proc/pressure files 24 # but throw errors when these files are opened. Catch these error 25 # and ensure that the reduce_proc_pressure directory is not created. 26 if os.path.exists("/proc/pressure"): 27 try: 28 with open('/proc/pressure/cpu', 'rb') as source: 29 source.read() 30 pressuredir = os.path.join(bsdir, 'reduced_proc_pressure') 31 bb.utils.mkdirhier(pressuredir) 32 file_handlers.extend([('pressure/cpu', self._reduce_pressure), 33 ('pressure/io', self._reduce_pressure), 34 ('pressure/memory', self._reduce_pressure)]) 35 except Exception: 36 pass 37 38 self.proc_files = [] 39 for filename, handler in (file_handlers): 40 # The corresponding /proc files might not exist on the host. 41 # For example, /proc/diskstats is not available in virtualized 42 # environments like Linux-VServer. Silently skip collecting 43 # the data. 44 if os.path.exists(os.path.join('/proc', filename)): 45 # In practice, this class gets instantiated only once in 46 # the bitbake cooker process. Therefore 'append' mode is 47 # not strictly necessary, but using it makes the class 48 # more robust should two processes ever write 49 # concurrently. 50 destfile = os.path.join(bsdir, '%sproc_%s.log' % ('reduced_' if handler else '', filename)) 51 self.proc_files.append((filename, open(destfile, 'ab'), handler)) 52 self.monitor_disk = open(os.path.join(bsdir, 'monitor_disk.log'), 'ab') 53 # Last time that we sampled /proc data resp. recorded disk monitoring data. 54 self.last_proc = 0 55 self.last_disk_monitor = 0 56 # Minimum number of seconds between recording a sample. This becames relevant when we get 57 # called very often while many short tasks get started. Sampling during quiet periods 58 # depends on the heartbeat event, which fires less often. 59 # By default, the Heartbeat events occur roughly once every second but the actual time 60 # between these events deviates by a few milliseconds, in most cases. Hence 61 # pick a somewhat arbitary tolerance such that we sample a large majority 62 # of the Heartbeat events. This ignores rare events that fall outside the minimum 63 # and may lead an extra sample in a given second every so often. However, it allows for fairly 64 # consistent intervals between samples without missing many events. 65 self.tolerance = 0.01 66 self.min_seconds = 1.0 - self.tolerance 67 68 self.meminfo_regex = re.compile(rb'^(MemTotal|MemFree|Buffers|Cached|SwapTotal|SwapFree):\s*(\d+)') 69 self.diskstats_regex = re.compile(rb'^([hsv]d.|mtdblock\d|mmcblk\d|cciss/c\d+d\d+.*)$') 70 self.diskstats_ltime = None 71 self.diskstats_data = None 72 self.stat_ltimes = None 73 # Last time we sampled /proc/pressure. All resources stored in a single dict with the key as filename 74 self.last_pressure = {"pressure/cpu": None, "pressure/io": None, "pressure/memory": None} 75 76 def close(self): 77 self.monitor_disk.close() 78 for _, output, _ in self.proc_files: 79 output.close() 80 81 def _reduce_meminfo(self, time, data, filename): 82 """ 83 Extracts 'MemTotal', 'MemFree', 'Buffers', 'Cached', 'SwapTotal', 'SwapFree' 84 and writes their values into a single line, in that order. 85 """ 86 values = {} 87 for line in data.split(b'\n'): 88 m = self.meminfo_regex.match(line) 89 if m: 90 values[m.group(1)] = m.group(2) 91 if len(values) == 6: 92 return (time, 93 b' '.join([values[x] for x in 94 (b'MemTotal', b'MemFree', b'Buffers', b'Cached', b'SwapTotal', b'SwapFree')]) + b'\n') 95 96 def _diskstats_is_relevant_line(self, linetokens): 97 if len(linetokens) != 14: 98 return False 99 disk = linetokens[2] 100 return self.diskstats_regex.match(disk) 101 102 def _reduce_diskstats(self, time, data, filename): 103 relevant_tokens = filter(self._diskstats_is_relevant_line, map(lambda x: x.split(), data.split(b'\n'))) 104 diskdata = [0] * 3 105 reduced = None 106 for tokens in relevant_tokens: 107 # rsect 108 diskdata[0] += int(tokens[5]) 109 # wsect 110 diskdata[1] += int(tokens[9]) 111 # use 112 diskdata[2] += int(tokens[12]) 113 if self.diskstats_ltime: 114 # We need to compute information about the time interval 115 # since the last sampling and record the result as sample 116 # for that point in the past. 117 interval = time - self.diskstats_ltime 118 if interval > 0: 119 sums = [ a - b for a, b in zip(diskdata, self.diskstats_data) ] 120 readTput = sums[0] / 2.0 * 100.0 / interval 121 writeTput = sums[1] / 2.0 * 100.0 / interval 122 util = float( sums[2] ) / 10 / interval 123 util = max(0.0, min(1.0, util)) 124 reduced = (self.diskstats_ltime, (readTput, writeTput, util)) 125 126 self.diskstats_ltime = time 127 self.diskstats_data = diskdata 128 return reduced 129 130 131 def _reduce_nop(self, time, data, filename): 132 return (time, data) 133 134 def _reduce_stat(self, time, data, filename): 135 if not data: 136 return None 137 # CPU times {user, nice, system, idle, io_wait, irq, softirq} from first line 138 tokens = data.split(b'\n', 1)[0].split() 139 times = [ int(token) for token in tokens[1:] ] 140 reduced = None 141 if self.stat_ltimes: 142 user = float((times[0] + times[1]) - (self.stat_ltimes[0] + self.stat_ltimes[1])) 143 system = float((times[2] + times[5] + times[6]) - (self.stat_ltimes[2] + self.stat_ltimes[5] + self.stat_ltimes[6])) 144 idle = float(times[3] - self.stat_ltimes[3]) 145 iowait = float(times[4] - self.stat_ltimes[4]) 146 147 aSum = max(user + system + idle + iowait, 1) 148 reduced = (time, (user/aSum, system/aSum, iowait/aSum)) 149 150 self.stat_ltimes = times 151 return reduced 152 153 def _reduce_pressure(self, time, data, filename): 154 """ 155 Return reduced pressure: {avg10, avg60, avg300} and delta total compared to the previous sample 156 for the cpu, io and memory resources. A common function is used for all 3 resources since the 157 format of the /proc/pressure file is the same in each case. 158 """ 159 if not data: 160 return None 161 tokens = data.split(b'\n', 1)[0].split() 162 avg10 = float(tokens[1].split(b'=')[1]) 163 avg60 = float(tokens[2].split(b'=')[1]) 164 avg300 = float(tokens[3].split(b'=')[1]) 165 total = int(tokens[4].split(b'=')[1]) 166 167 reduced = None 168 if self.last_pressure[filename]: 169 delta = total - self.last_pressure[filename] 170 reduced = (time, (avg10, avg60, avg300, delta)) 171 self.last_pressure[filename] = total 172 return reduced 173 174 def sample(self, event, force): 175 """ 176 Collect and log proc or disk_monitor stats periodically. 177 Return True if a new sample is collected and hence the value last_proc or last_disk_monitor 178 is changed. 179 """ 180 retval = False 181 now = time.time() 182 if (now - self.last_proc > self.min_seconds) or force: 183 for filename, output, handler in self.proc_files: 184 with open(os.path.join('/proc', filename), 'rb') as input: 185 data = input.read() 186 if handler: 187 reduced = handler(now, data, filename) 188 else: 189 reduced = (now, data) 190 if reduced: 191 if isinstance(reduced[1], bytes): 192 # Use as it is. 193 data = reduced[1] 194 else: 195 # Convert to a single line. 196 data = (' '.join([str(x) for x in reduced[1]]) + '\n').encode('ascii') 197 # Unbuffered raw write, less overhead and useful 198 # in case that we end up with concurrent writes. 199 os.write(output.fileno(), 200 ('%.0f\n' % reduced[0]).encode('ascii') + 201 data + 202 b'\n') 203 self.last_proc = now 204 retval = True 205 206 if isinstance(event, bb.event.MonitorDiskEvent) and \ 207 ((now - self.last_disk_monitor > self.min_seconds) or force): 208 os.write(self.monitor_disk.fileno(), 209 ('%.0f\n' % now).encode('ascii') + 210 ''.join(['%s: %d\n' % (dev, sample.total_bytes - sample.free_bytes) 211 for dev, sample in event.disk_usage.items()]).encode('ascii') + 212 b'\n') 213 self.last_disk_monitor = now 214 retval = True 215 return retval