diff --git a/README.md b/README.md index 5f744db..7dd2688 100644 --- a/README.md +++ b/README.md @@ -89,7 +89,7 @@ sudo launchctl unload /Library/LaunchDaemons/berlin.green-coding.hog.plist ### Settings It is possible to configure your own settings by using a `settings.ini` file in the same directory as the `power_logger.py` -script or adding a `.hog_settings.ini` to your home folder. The home folder settings will be prioritized. +script. Or adding a `hog_settings.ini` file to `/etc/`. This will file will be prioritized. Following keys are currently used: @@ -108,7 +108,8 @@ The hog desktop app gives you analytics of the data that was recorded. Please mo new process which will all show up in the coalition. Sometimes a shell might turn up here. Please tell us so we can add this as an exception - `Energy Impact`: This is the value mac gives it's processes. The exact formula is not known but we know that quite some - factors are considered. For now this is the best value we've got 🫣 + factors are considered [some details](https://blog.mozilla.org/nnethercote/2015/08/26/what-does-the-os-x-activity-monitors-energy-impact-actually-measure/). + For now this is the best value we've got 🫣 - `AVG Cpu Time %`: This is how long this coalition has spent on the CPUs. We take a percentage which can be over 100% as the coalition could run on multiple cpus at the same time. So if a process takes up 100% of cpu time and runs on 4 cpus the time will be 400%. diff --git a/app/hog/hog.xcodeproj/project.pbxproj b/app/hog/hog.xcodeproj/project.pbxproj index 45a5127..84526f4 100644 --- a/app/hog/hog.xcodeproj/project.pbxproj +++ b/app/hog/hog.xcodeproj/project.pbxproj @@ -434,7 +434,7 @@ "CODE_SIGN_IDENTITY[sdk=macosx*]" = "Apple Development"; CODE_SIGN_STYLE = Automatic; COMBINE_HIDPI_IMAGES = YES; - CURRENT_PROJECT_VERSION = 1; + CURRENT_PROJECT_VERSION = 2; DEAD_CODE_STRIPPING = YES; DEVELOPMENT_ASSET_PATHS = "\"hog/Preview Content\""; DEVELOPMENT_TEAM = SBWA476E6F; @@ -468,7 +468,7 @@ "CODE_SIGN_IDENTITY[sdk=macosx*]" = "Apple Development"; CODE_SIGN_STYLE = Automatic; COMBINE_HIDPI_IMAGES = YES; - CURRENT_PROJECT_VERSION = 1; + CURRENT_PROJECT_VERSION = 2; DEAD_CODE_STRIPPING = YES; DEVELOPMENT_ASSET_PATHS = "\"hog/Preview Content\""; DEVELOPMENT_TEAM = SBWA476E6F; diff --git a/app/hog/hog.xcodeproj/project.xcworkspace/xcuserdata/didi.xcuserdatad/UserInterfaceState.xcuserstate b/app/hog/hog.xcodeproj/project.xcworkspace/xcuserdata/didi.xcuserdatad/UserInterfaceState.xcuserstate index 2c9ebbc..00be01d 100644 Binary files a/app/hog/hog.xcodeproj/project.xcworkspace/xcuserdata/didi.xcuserdatad/UserInterfaceState.xcuserstate and b/app/hog/hog.xcodeproj/project.xcworkspace/xcuserdata/didi.xcuserdatad/UserInterfaceState.xcuserstate differ diff --git a/app/hog/hog/Assets.xcassets/Colors/red.colorset/Contents.json b/app/hog/hog/Assets.xcassets/Colors/redish.colorset/Contents.json similarity index 100% rename from app/hog/hog/Assets.xcassets/Colors/red.colorset/Contents.json rename to app/hog/hog/Assets.xcassets/Colors/redish.colorset/Contents.json diff --git a/app/hog/hog/DetailView.swift b/app/hog/hog/DetailView.swift index 3d90753..3049faf 100644 --- a/app/hog/hog/DetailView.swift +++ b/app/hog/hog/DetailView.swift @@ -228,14 +228,14 @@ class SettingsManager: ObservableObject { class ValueManager: ObservableObject { var lookBackTime:Int = 0 - @Published var energy: CGFloat = 0 + @Published var energy: Int64 = 0 @Published var providerRunning: Bool = false @Published var topApp: String = "Loading..." @Published var isLoading: Bool = true enum ValueType { - case float + case int case string } @@ -258,14 +258,14 @@ class ValueManager: ObservableObject { return } - var newEnergy: CGFloat = 0 + var newEnergy: Int64 = 0 var energyQuery:String if self.lookBackTime == 0 { energyQuery = "SELECT COALESCE(sum(combined_energy), 0) FROM power_measurements;" }else{ energyQuery = "SELECT COALESCE(sum(combined_energy), 0) FROM power_measurements WHERE time >= ((CAST(strftime('%s', 'now') AS INTEGER) * 1000) - \(self.lookBackTime));" } - if let result: CGFloat = queryDatabase(db: db, query:energyQuery, type: .float) { + if let result: Int64 = queryDatabase(db: db, query:energyQuery, type: .int) { newEnergy = result } @@ -318,8 +318,8 @@ class ValueManager: ObservableObject { if sqlite3_prepare_v2(db, query, -1, &queryStatement, nil) == SQLITE_OK { if sqlite3_step(queryStatement) == SQLITE_ROW { switch type { - case .float: - let value = CGFloat(sqlite3_column_double(queryStatement, 0)) + case .int: + let value = sqlite3_column_int64(queryStatement, 0) sqlite3_finalize(queryStatement) return value as? T case .string: @@ -338,7 +338,7 @@ class ValueManager: ObservableObject { struct TopProcess: Codable, Identifiable { let id: UUID = UUID() // Add this line if you want a unique identifier let name: String - let energy_impact: Double + let energy_impact: Int64 let cputime_per: Int32 enum CodingKeys: String, CodingKey { @@ -426,7 +426,7 @@ class TopProcessData: Identifiable, ObservableObject, RandomAccessCollection { if let namePointer = sqlite3_column_text(queryStatement, 0) { name = String(cString: namePointer) } - let energy_impact = sqlite3_column_double(queryStatement, 1) + let energy_impact = sqlite3_column_int64(queryStatement, 1) let cputime_per = sqlite3_column_int(queryStatement, 2) newLines.append(TopProcess(name: name, energy_impact: energy_impact, cputime_per: cputime_per)) @@ -447,11 +447,11 @@ class TopProcessData: Identifiable, ObservableObject, RandomAccessCollection { struct DataPoint: Codable, Identifiable { - let id: Double - let combined_energy: Double - let cpu_energy: Double - let gpu_energy: Double - let ane_energy: Double + let id: Int64 + let combined_energy: Int64 + let cpu_energy: Int64 + let gpu_energy: Int64 + let ane_energy: Int64 var time: Date? enum CodingKeys: String, CodingKey { @@ -507,12 +507,12 @@ class ChartData: ObservableObject, RandomAccessCollection { if sqlite3_prepare_v2(db, queryString, -1, &queryStatement, nil) == SQLITE_OK { var newPoints: [DataPoint] = [] while sqlite3_step(queryStatement) == SQLITE_ROW { - let id = sqlite3_column_double(queryStatement, 0) - let combined_energy = sqlite3_column_double(queryStatement, 2) - let cpu_energy = sqlite3_column_double(queryStatement, 3) - let gpu_energy = sqlite3_column_double(queryStatement, 4) - let ane_energy = sqlite3_column_double(queryStatement, 5) - let time = Date(timeIntervalSince1970: id / 1000.0) + let id = sqlite3_column_int64(queryStatement, 0) + let combined_energy = sqlite3_column_int64(queryStatement, 2) + let cpu_energy = sqlite3_column_int64(queryStatement, 3) + let gpu_energy = sqlite3_column_int64(queryStatement, 4) + let ane_energy = sqlite3_column_int64(queryStatement, 5) + let time = Date(timeIntervalSince1970: Double(id) / 1000.0) let dataPoint = DataPoint(id: id, combined_energy: combined_energy, cpu_energy: cpu_energy, gpu_energy: gpu_energy, ane_energy: ane_energy, time: time) @@ -599,7 +599,7 @@ struct TopProcessTable: View { TableColumn("Name", value: \TopProcess.name) TableColumn("Energy Impact", value: \TopProcess.energy_impact){ line in - Text(String(format: "%.0f", line.energy_impact)) + Text(String(line.energy_impact)) } TableColumn("AVG CPU time %", value: \TopProcess.cputime_per){ line in Text(String(line.cputime_per)) @@ -696,25 +696,24 @@ struct DataView: View { TextBadge(title: "", color: Color("chartColor2"), image: "checkmark.seal", value: "All measurement systems are functional") } else { HStack{ - TextBadge(title: "", color: Color("red"), image: "exclamationmark.octagon", value: "Measurement systems not running!") + TextBadge(title: "", color: Color("redish"), image: "exclamationmark.octagon", value: "Measurement systems not running!") Link(destination: URL(string: "https://github.com/green-coding-berlin/hog#power-logger")!) { Image(systemName: "questionmark.circle.fill") .font(.system(size: 24)) } } } - HStack{ - TextBadge(title: "", color: Color("menuTab"), image: "person.crop.circle.badge.clock", value: "No project set") - Button(action: { - isTextInputViewPresented = true - }) { - Image(systemName: "pencil.circle") - } - } - .sheet(isPresented: $isTextInputViewPresented) { - TextInputView(text: $text, isPresented: $isTextInputViewPresented) - } - +// HStack{ +// TextBadge(title: "", color: Color("menuTab"), image: "person.crop.circle.badge.clock", value: "No project set") +// Button(action: { +// isTextInputViewPresented = true +// }) { +// Image(systemName: "pencil.circle") +// } +// } +// .sheet(isPresented: $isTextInputViewPresented) { +// TextInputView(text: $text, isPresented: $isTextInputViewPresented) +// } } Button(action: { if let url = URL(string: "\(settingsManager.web_url)\(settingsManager.machine_uuid)") { @@ -756,8 +755,8 @@ func ProcessBadge(title: String, color: Color, process: String)->some View { .frame(maxWidth: .infinity, alignment: .leading) } -func formatEnergy(_ mJ: Double) -> String { - let joules = mJ / 1000.0 +func formatEnergy(_ mJ: Int64) -> String { + let joules = Double(mJ) / 1000.0 let wattHours = joules / 3600.0 let wattMinutes = joules / 60.0 @@ -769,8 +768,9 @@ func formatEnergy(_ mJ: Double) -> String { } + @ViewBuilder -func EnergyBadge(title: String, color: Color, image: String, value: CGFloat)->some View { +func EnergyBadge(title: String, color: Color, image: String, value: Int64)->some View { HStack { Image(systemName: image) .font(.title2) diff --git a/hog.app/Contents/Info.plist b/hog.app/Contents/Info.plist index d441945..a13ab0f 100644 --- a/hog.app/Contents/Info.plist +++ b/hog.app/Contents/Info.plist @@ -29,7 +29,7 @@ MacOSX CFBundleVersion - 1 + 2 DTCompiler com.apple.compilers.llvm.clang.1_0 DTPlatformBuild diff --git a/hog.app/Contents/MacOS/hog b/hog.app/Contents/MacOS/hog index da3deb1..0a71938 100755 Binary files a/hog.app/Contents/MacOS/hog and b/hog.app/Contents/MacOS/hog differ diff --git a/hog.app/Contents/Resources/Assets.car b/hog.app/Contents/Resources/Assets.car index b5ba2f8..33f013a 100644 Binary files a/hog.app/Contents/Resources/Assets.car and b/hog.app/Contents/Resources/Assets.car differ diff --git a/hog.app/Contents/_CodeSignature/CodeResources b/hog.app/Contents/_CodeSignature/CodeResources index 8cc7ec0..e8f3cd8 100644 --- a/hog.app/Contents/_CodeSignature/CodeResources +++ b/hog.app/Contents/_CodeSignature/CodeResources @@ -10,7 +10,7 @@ Resources/Assets.car - KVS4Oun43bgfxB3YkNfdhw1dl3g= + ZzCJz9XoXNSp0RECXMKKF0A0qsY= Resources/demo_db.db @@ -30,7 +30,7 @@ hash2 - Tyc2sRL+1ilGRhjOAlNflHIsFpR7wKLVWqMqs6PBik8= + nSTOBfK1SHjTRnjJWSlyNElvk/MVGWyMvAORkrjjXsE= Resources/demo_db.db diff --git a/install.sh b/install.sh index d1230e5..edb9a02 100644 --- a/install.sh +++ b/install.sh @@ -33,7 +33,7 @@ install_xcode_clt() { # Call the function to ensure Xcode Command Line Tools are installed install_xcode_clt -ZIP_LOCATION=$(curl -s https://api.github.com/repos/green-coding-berlin/hog/releases/latest | grep '/hog_power_logger.zip' | cut -d\" -f4) +ZIP_LOCATION=$(curl -s https://api.github.com/repos/green-coding-berlin/hog/releases/latest | grep -o 'https://[^"]*/hog_power_logger.zip’) curl -fLo /tmp/latest_release.zip $ZIP_LOCATION mkdir -p /usr/local/bin/hog diff --git a/metrics_error_finder.py b/metrics_error_finder.py new file mode 100755 index 0000000..dab597a --- /dev/null +++ b/metrics_error_finder.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 + +# pylint: disable=W0603,W0602 +import json +import subprocess +import time +import plistlib +import argparse +import zlib +import base64 +import xml +import signal +import sys +import uuid +import os +import os.path +import stat +import urllib.request +import configparser +import sqlite3 +import http +from datetime import timezone +from pathlib import Path + +from libs import caribou + + +# Shared variable to signal the thread to stop +stop_signal = False + + +def sigint_handler(_, __): + global stop_signal + if stop_signal: + # If you press CTR-C the second time we bail + sys.exit() + + stop_signal = True + print('Received stop signal. Terminating all processes.') + +def siginfo_handler(_, __): + print(SETTINGS) + print(stats) + +signal.signal(signal.SIGINT, sigint_handler) +signal.signal(signal.SIGTERM, sigint_handler) + +signal.signal(signal.SIGINFO, siginfo_handler) + + + +SETTINGS = { + 'powermetrics': 5000, +} + + +def run_powermetrics(): + + cmd = ['powermetrics', + '--show-all', + '-i', str(SETTINGS['powermetrics']), + '-f', 'plist'] + + with subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True) as process: + buffer = [] + for line in process.stdout: + line = line.strip().replace('&', '&') + buffer.append(line) + if line == '': + parse_powermetrics_output(''.join(buffer)) + buffer = [] + + if stop_signal: + break + + if stop_signal: + process.terminate() + +def find_top_processes(data: list): + # As iterm2 will probably show up as it spawns the processes called from the shell we look at the tasks + new_data = [] + for coalition in data: + if coalition['name'] == 'com.googlecode.iterm2' or coalition['name'].strip() == '': + new_data.extend(coalition['tasks']) + else: + new_data.append(coalition) + + return new_data + +def is_difference_more_than_5_percent(x, y): + if x == 0 and y == 0: + return False # If both values are 0, the percentage difference is undefined. + + if x == 0 or y == 0: + return True # If one of the values is 0 and the other is not, they differ by more than 5%. + + percent_difference = abs(x - y) / ((x + y) / 2) * 100 + + return percent_difference > 5 + + +def parse_powermetrics_output(output: str): + global stats + + for data in output.encode('utf-8').split(b'\x00'): + if data: + + if data == b'powermetrics must be invoked as the superuser\n': + raise PermissionError('You need to run this script as root!') + + try: + data=plistlib.loads(data) + data['timezone'] = time.tzname + data['timestamp'] = int(data['timestamp'].replace(tzinfo=timezone.utc).timestamp() * 1e3) + except xml.parsers.expat.ExpatError as exc: + print(data) + raise exc + + for process in find_top_processes(data['coalitions']): + + cpu_ns_dirty = process['cputime_ns'] + cpu_ns_clean = ((process['cputime_ms_per_s'] * 1_000_000) / 1_000_000_000) * data['elapsed_ns'] + + ei_dirty = process['energy_impact'] + ei_clean = process['energy_impact_per_s'] * data['elapsed_ns'] / 1_000_000_000 + + if is_difference_more_than_5_percent(cpu_ns_dirty, cpu_ns_clean) or \ + is_difference_more_than_5_percent(ei_dirty, ei_clean): + + print(f"Name : {process['name']}") + print(f"Elapsed ns : {data['elapsed_ns']}") + print('') + print(f"CPU Time ns : {process['cputime_ns']}") + print(f"CPU Time ns / con : {cpu_ns_clean}") + print(f"cputime_ms_per_s : {process['cputime_ms_per_s']}") + print('') + print(f"energy_impact : {process['energy_impact']}") + print(f"energy_impact con : {ei_clean}") + print(f"energy_impact_per_s : {process['energy_impact_per_s']}") + print('') + print(f"diskio_bytesread : {process['diskio_bytesread']}") + print(f"diskio_bytesread_per_s: {process['diskio_bytesread_per_s']}") + print('') + print(process) + print('------------') + +if __name__ == '__main__': + run_powermetrics() diff --git a/migrations/20230909161250_first_db.py b/migrations/20230909161250_first_db.py index 059718c..d722966 100644 --- a/migrations/20230909161250_first_db.py +++ b/migrations/20230909161250_first_db.py @@ -19,11 +19,11 @@ def upgrade(connection): cpu_energy INT, gpu_energy INT, ane_energy INT, - energy_impact REAL)''' + energy_impact INT)''' connection.execute(tbl_power_measurements) tbl_top_processes = '''CREATE TABLE IF NOT EXISTS top_processes - (time INT, name STRING, energy_impact REAL, cputime_per INT)''' + (time INT, name STRING, energy_impact INT, cputime_per INT)''' connection.execute(tbl_top_processes) tbl_settings = '''CREATE TABLE IF NOT EXISTS settings diff --git a/power_logger.py b/power_logger.py index 9f8e4c8..229bc07 100755 --- a/power_logger.py +++ b/power_logger.py @@ -29,7 +29,7 @@ stop_signal = False stats = { - 'combined_power': 0, + 'combined_energy': 0, 'cpu_energy': 0, 'gpu_energy': 0, 'ane_energy': 0, @@ -71,11 +71,10 @@ def siginfo_handler(_, __): 'upload_data': True, } -home_dir = os.path.expanduser('~') script_dir = os.path.dirname(os.path.realpath(__file__)) -if os.path.exists(os.path.join(home_dir, '.hog_settings.ini')): - config_path = os.path.join(home_dir, '.hog_settings.ini') +if os.path.exists('/etc/hog_settings.ini'): + config_path = '/etc/hog_settings.ini' elif os.path.exists(os.path.join(script_dir, 'settings.ini')): config_path = os.path.join(script_dir, 'settings.ini') else: @@ -185,7 +184,7 @@ def upload_data_to_endpoint(): method='POST') try: with urllib.request.urlopen(req) as response: - if response.status == 200: + if response.status == 204: for p in payload: c.execute('UPDATE measurements SET uploaded = ?, data = NULL WHERE id = ?;', (int(time.time()), p['row_id'])) conn.commit() @@ -199,7 +198,7 @@ def upload_data_to_endpoint(): break -def find_top_processes(data: list): +def find_top_processes(data: list, elapsed_ns:int): # As iterm2 will probably show up as it spawns the processes called from the shell we look at the tasks new_data = [] for coalition in data: @@ -211,8 +210,10 @@ def find_top_processes(data: list): for p in sorted(new_data, key=lambda k: k['energy_impact'], reverse=True)[:10]: yield{ 'name': p['name'], - 'energy_impact': p['energy_impact'], - 'cputime_ns': p['cputime_ns'] + # Energy_impact and cputime are broken so we need to use the per_s and convert them + # Check the https://www.green-coding.berlin/blog/ for details + 'energy_impact': round((p['energy_impact_per_s'] / 1_000_000_000) * elapsed_ns), + 'cputime_ns': ((p['cputime_ms_per_s'] * 1_000_000) / 1_000_000_000) * elapsed_ns, } @@ -240,29 +241,30 @@ def parse_powermetrics_output(output: str): (data['timestamp'], compressed_data_str)) cpu_energy_data = {} + energy_impact = round(data['all_tasks'].get('energy_impact_per_s') * data['elapsed_ns'] / 1_000_000_000) if 'ane_energy' in data['processor']: cpu_energy_data = { - 'combined_power': int(data['processor'].get('combined_power', 0) * data['elapsed_ns'] / 1_000_000_000.0), - 'cpu_energy': int(data['processor'].get('cpu_energy', 0)), - 'gpu_energy': int(data['processor'].get('gpu_energy', 0)), - 'ane_energy': int(data['processor'].get('ane_energy', 0)), - 'energy_impact': data['all_tasks'].get('energy_impact'), + 'combined_energy': round(data['processor'].get('combined_power', 0) * data['elapsed_ns'] / 1_000_000_000.0), + 'cpu_energy': round(data['processor'].get('cpu_energy', 0)), + 'gpu_energy': round(data['processor'].get('gpu_energy', 0)), + 'ane_energy': round(data['processor'].get('ane_energy', 0)), + 'energy_impact': energy_impact, } elif 'package_joules' in data['processor']: # Intel processors report in joules/ watts and not mJ cpu_energy_data = { - 'combined_power': int(data['processor'].get('package_joules', 0) * 1_000), - 'cpu_energy': int(data['processor'].get('cpu_joules', 0) * 1_000), - 'gpu_energy': int(data['processor'].get('igpu_watts', 0) * data['elapsed_ns'] / 1_000_000_000.0 * 1_000), + 'combined_energy': round(data['processor'].get('package_joules', 0) * 1_000), + 'cpu_energy': round(data['processor'].get('cpu_joules', 0) * 1_000), + 'gpu_energy': round(data['processor'].get('igpu_watts', 0) * data['elapsed_ns'] / 1_000_000_000.0 * 1_000), 'ane_energy': 0, - 'energy_impact': data['all_tasks'].get('energy_impact'), + 'energy_impact': energy_impact, } c.execute('''INSERT INTO power_measurements (time, combined_energy, cpu_energy, gpu_energy, ane_energy, energy_impact) VALUES (?, ?, ?, ?, ?, ?)''', (data['timestamp'], - cpu_energy_data['combined_power'], + cpu_energy_data['combined_energy'], cpu_energy_data['cpu_energy'], cpu_energy_data['gpu_energy'], cpu_energy_data['ane_energy'], @@ -272,7 +274,7 @@ def parse_powermetrics_output(output: str): stats[key] += cpu_energy_data[key] - for process in find_top_processes(data['coalitions']): + for process in find_top_processes(data['coalitions'], data['elapsed_ns']): cpu_per = int(process['cputime_ns'] / data['elapsed_ns'] * 100) c.execute('INSERT INTO top_processes (time, name, energy_impact, cputime_per) VALUES (?, ?, ?, ?)', (data['timestamp'], process['name'], process['energy_impact'], cpu_per))