A fork of https://github.com/crosspoint-reader/crosspoint-reader
feat: Extend python debugging monitor functionality (keyword filter / suppress) (#810)
## Summary
* I needed the ability to filter and or suppress debug messages
containig certain keywords (eg [GFX] for render related stuff)
* Update of debugging_monitor.py script for development work
## Additional Context
```
usage: debugging_monitor.py [-h] [--baud BAUD] [--filter FILTER] [--suppress SUPPRESS] [port]
ESP32 Monitor with Graph
positional arguments:
port Serial port
options:
-h, --help show this help message and exit
--baud BAUD Baud rate
--filter FILTER Only display lines containing this keyword (case-insensitive)
--suppress SUPPRESS Suppress lines containing this keyword (case-insensitive)
```
* plus a couple of platform specific defaults (port, pip style)
---
### AI Usage
While CrossPoint doesn't have restrictions on AI tools in contributing,
please be transparent about their usage as it
helps set the right context for reviewers.
Did you use AI tools to help write this code? NO
+206
-75
+206
-75
scripts/debugging_monitor.py
+206
-75
scripts/debugging_monitor.py
···
1
+
#!/usr/bin/env python3
2
+
"""
3
+
ESP32 Serial Monitor with Memory Graph
4
+
5
+
This script provides a real-time serial monitor for ESP32 devices with
6
+
integrated memory usage graphing capabilities. It reads serial output,
7
+
parses memory information, and displays it in both console and graphical form.
8
+
"""
9
+
1
10
import sys
2
11
import argparse
3
12
import re
4
13
import threading
5
14
from datetime import datetime
6
15
from collections import deque
7
-
import time
8
16
9
17
# Try to import potentially missing packages
18
+
PACKAGE_MAPPING: dict[str, str] = {
19
+
"serial": "pyserial",
20
+
"colorama": "colorama",
21
+
"matplotlib": "matplotlib",
22
+
}
23
+
10
24
try:
11
25
import serial
12
26
from colorama import init, Fore, Style
13
27
import matplotlib.pyplot as plt
14
-
import matplotlib.animation as animation
28
+
from matplotlib import animation
15
29
except ImportError as e:
16
-
missing_package = e.name
30
+
ERROR_MSG = str(e).lower()
31
+
missing_packages = [pkg for mod, pkg in PACKAGE_MAPPING.items() if mod in ERROR_MSG]
32
+
33
+
if not missing_packages:
34
+
# Fallback if mapping doesn't cover
35
+
missing_packages = ["pyserial", "colorama", "matplotlib"]
36
+
17
37
print("\n" + "!" * 50)
18
-
print(f" Error: The required package '{missing_package}' is not installed.")
38
+
print(f" Error: Required package(s) not installed: {', '.join(missing_packages)}")
19
39
print("!" * 50)
20
40
21
-
print(f"\nTo fix this, please run the following command in your terminal:\n")
22
-
23
-
install_cmd = "pip install "
24
-
packages = []
25
-
if 'serial' in str(e): packages.append("pyserial")
26
-
if 'colorama' in str(e): packages.append("colorama")
27
-
if 'matplotlib' in str(e): packages.append("matplotlib")
28
-
29
-
print(f" {install_cmd}{' '.join(packages)}")
41
+
print("\nTo fix this, please run the following command in your terminal:\n")
42
+
INSTALL_CMD = "pip install " if sys.platform.startswith("win") else "pip3 install "
43
+
print(f" {INSTALL_CMD}{' '.join(missing_packages)}")
30
44
31
45
print("\nExiting...")
32
46
sys.exit(1)
···
34
48
# --- Global Variables for Data Sharing ---
35
49
# Store last 50 data points
36
50
MAX_POINTS = 50
37
-
time_data = deque(maxlen=MAX_POINTS)
38
-
free_mem_data = deque(maxlen=MAX_POINTS)
39
-
total_mem_data = deque(maxlen=MAX_POINTS)
40
-
data_lock = threading.Lock() # Prevent reading while writing
51
+
time_data: deque[str] = deque(maxlen=MAX_POINTS)
52
+
free_mem_data: deque[float] = deque(maxlen=MAX_POINTS)
53
+
total_mem_data: deque[float] = deque(maxlen=MAX_POINTS)
54
+
data_lock: threading.Lock = threading.Lock() # Prevent reading while writing
41
55
42
56
# Initialize colors
43
57
init(autoreset=True)
44
58
45
-
def get_color_for_line(line):
59
+
# Color mapping for log lines
60
+
COLOR_KEYWORDS: dict[str, list[str]] = {
61
+
Fore.RED: ["ERROR", "[ERR]", "[SCT]", "FAILED", "WARNING"],
62
+
Fore.CYAN: ["[MEM]", "FREE:"],
63
+
Fore.MAGENTA: [
64
+
"[GFX]",
65
+
"[ERS]",
66
+
"DISPLAY",
67
+
"RAM WRITE",
68
+
"RAM COMPLETE",
69
+
"REFRESH",
70
+
"POWERING ON",
71
+
"FRAME BUFFER",
72
+
"LUT",
73
+
],
74
+
Fore.GREEN: [
75
+
"[EBP]",
76
+
"[BMC]",
77
+
"[ZIP]",
78
+
"[PARSER]",
79
+
"[EHP]",
80
+
"LOADING EPUB",
81
+
"CACHE",
82
+
"DECOMPRESSED",
83
+
"PARSING",
84
+
],
85
+
Fore.YELLOW: ["[ACT]", "ENTERING ACTIVITY", "EXITING ACTIVITY"],
86
+
Fore.BLUE: ["RENDERED PAGE", "[LOOP]", "DURATION", "WAIT COMPLETE"],
87
+
Fore.LIGHTYELLOW_EX: [
88
+
"[CPS]",
89
+
"SETTINGS",
90
+
"[CLEAR_CACHE]",
91
+
"[CHAP]",
92
+
"[OPDS]",
93
+
"[COF]",
94
+
],
95
+
Fore.LIGHTBLACK_EX: [
96
+
"ESP-ROM",
97
+
"BUILD:",
98
+
"RST:",
99
+
"BOOT:",
100
+
"SPIWP:",
101
+
"MODE:",
102
+
"LOAD:",
103
+
"ENTRY",
104
+
"[SD]",
105
+
"STARTING CROSSPOINT",
106
+
"VERSION",
107
+
],
108
+
Fore.LIGHTCYAN_EX: ["[RBS]"],
109
+
Fore.LIGHTMAGENTA_EX: [
110
+
"[KRS]",
111
+
"EINKDISPLAY:",
112
+
"STATIC FRAME",
113
+
"INITIALIZING",
114
+
"SPI INITIALIZED",
115
+
"GPIO PINS",
116
+
"RESETTING",
117
+
"SSD1677",
118
+
"E-INK",
119
+
],
120
+
Fore.LIGHTGREEN_EX: ["[FNS]", "FOOTNOTE"],
121
+
}
122
+
123
+
124
+
# pylint: disable=R0912
125
+
def get_color_for_line(line: str) -> str:
46
126
"""
47
127
Classify log lines by type and assign appropriate colors.
48
128
"""
49
129
line_upper = line.upper()
130
+
for color, keywords in COLOR_KEYWORDS.items():
131
+
if any(keyword in line_upper for keyword in keywords):
132
+
return color
133
+
return Fore.WHITE
50
134
51
-
if any(keyword in line_upper for keyword in ["ERROR", "[ERR]", "[SCT]", "FAILED", "WARNING"]):
52
-
return Fore.RED
53
-
if "[MEM]" in line_upper or "FREE:" in line_upper:
54
-
return Fore.CYAN
55
-
if any(keyword in line_upper for keyword in ["[GFX]", "[ERS]", "DISPLAY", "RAM WRITE", "RAM COMPLETE", "REFRESH", "POWERING ON", "FRAME BUFFER", "LUT"]):
56
-
return Fore.MAGENTA
57
-
if any(keyword in line_upper for keyword in ["[EBP]", "[BMC]", "[ZIP]", "[PARSER]", "[EHP]", "LOADING EPUB", "CACHE", "DECOMPRESSED", "PARSING"]):
58
-
return Fore.GREEN
59
-
if "[ACT]" in line_upper or "ENTERING ACTIVITY" in line_upper or "EXITING ACTIVITY" in line_upper:
60
-
return Fore.YELLOW
61
-
if any(keyword in line_upper for keyword in ["RENDERED PAGE", "[LOOP]", "DURATION", "WAIT COMPLETE"]):
62
-
return Fore.BLUE
63
-
if any(keyword in line_upper for keyword in ["[CPS]", "SETTINGS", "[CLEAR_CACHE]"]):
64
-
return Fore.LIGHTYELLOW_EX
65
-
if any(keyword in line_upper for keyword in ["ESP-ROM", "BUILD:", "RST:", "BOOT:", "SPIWP:", "MODE:", "LOAD:", "ENTRY", "[SD]", "STARTING CROSSPOINT", "VERSION"]):
66
-
return Fore.LIGHTBLACK_EX
67
-
if "[RBS]" in line_upper:
68
-
return Fore.LIGHTCYAN_EX
69
-
if "[KRS]" in line_upper:
70
-
return Fore.LIGHTMAGENTA_EX
71
-
if any(keyword in line_upper for keyword in ["EINKDISPLAY:", "STATIC FRAME", "INITIALIZING", "SPI INITIALIZED", "GPIO PINS", "RESETTING", "SSD1677", "E-INK"]):
72
-
return Fore.LIGHTMAGENTA_EX
73
-
if any(keyword in line_upper for keyword in ["[FNS]", "FOOTNOTE"]):
74
-
return Fore.LIGHTGREEN_EX
75
-
if any(keyword in line_upper for keyword in ["[CHAP]", "[OPDS]", "[COF]"]):
76
-
return Fore.LIGHTYELLOW_EX
77
135
78
-
return Fore.WHITE
79
-
80
-
def parse_memory_line(line):
136
+
def parse_memory_line(line: str) -> tuple[int | None, int | None]:
81
137
"""
82
138
Extracts Free and Total bytes from the specific log line.
83
139
Format: [MEM] Free: 196344 bytes, Total: 226412 bytes, Min Free: 112620 bytes
···
93
149
return None, None
94
150
return None, None
95
151
96
-
def serial_worker(port, baud):
152
+
153
+
def serial_worker(port: str, baud: int, kwargs: dict[str, str]) -> None:
97
154
"""
98
155
Runs in a background thread. Handles reading serial, printing to console,
99
156
and updating the data lists.
100
157
"""
101
158
print(f"{Fore.CYAN}--- Opening {port} at {baud} baud ---{Style.RESET_ALL}")
159
+
filter_keyword = kwargs.get("filter", "").lower()
160
+
suppress = kwargs.get("suppress", "").lower()
161
+
if filter_keyword and suppress and filter_keyword == suppress:
162
+
print(
163
+
f"{Fore.YELLOW}Warning: Filter and Suppress keywords are the same. "
164
+
f"This may result in no output.{Style.RESET_ALL}"
165
+
)
166
+
if filter_keyword:
167
+
print(
168
+
f"{Fore.YELLOW}Filtering lines to only show those containing: "
169
+
f"'{filter_keyword}'{Style.RESET_ALL}"
170
+
)
171
+
if suppress:
172
+
print(
173
+
f"{Fore.YELLOW}Suppressing lines containing: '{suppress}'{Style.RESET_ALL}"
174
+
)
102
175
103
176
try:
104
177
ser = serial.Serial(port, baud, timeout=0.1)
···
111
184
try:
112
185
while True:
113
186
try:
114
-
raw_data = ser.readline().decode('utf-8', errors='replace')
187
+
raw_data = ser.readline().decode("utf-8", errors="replace")
115
188
116
189
if not raw_data:
117
190
continue
···
127
200
# Check for Memory Line
128
201
if "[MEM]" in formatted_line:
129
202
free_val, total_val = parse_memory_line(formatted_line)
130
-
if free_val is not None:
203
+
if free_val is not None and total_val is not None:
131
204
with data_lock:
132
205
time_data.append(pc_time)
133
-
free_mem_data.append(free_val / 1024) # Convert to KB
134
-
total_mem_data.append(total_val / 1024) # Convert to KB
135
-
206
+
free_mem_data.append(free_val / 1024) # Convert to KB
207
+
total_mem_data.append(total_val / 1024) # Convert to KB
208
+
# Apply filters
209
+
if filter_keyword and filter_keyword not in formatted_line.lower():
210
+
continue
211
+
if suppress and suppress in formatted_line.lower():
212
+
continue
136
213
# Print to console
137
214
line_color = get_color_for_line(formatted_line)
138
215
print(f"{line_color}{formatted_line}")
139
216
140
-
except OSError:
141
-
print(f"{Fore.RED}Device disconnected.{Style.RESET_ALL}")
217
+
except (OSError, UnicodeDecodeError):
218
+
print(f"{Fore.RED}Device disconnected or data error.{Style.RESET_ALL}")
142
219
break
143
-
except Exception as e:
220
+
except KeyboardInterrupt:
144
221
# If thread is killed violently (e.g. main exit), silence errors
145
222
pass
146
223
finally:
147
-
if 'ser' in locals() and ser.is_open:
224
+
if "ser" in locals() and ser.is_open:
148
225
ser.close()
149
226
150
-
def update_graph(frame):
227
+
228
+
def update_graph(frame) -> list: # pylint: disable=unused-argument
151
229
"""
152
230
Called by Matplotlib animation to redraw the chart.
153
231
"""
154
232
with data_lock:
155
233
if not time_data:
156
-
return
234
+
return []
157
235
158
236
# Convert deques to lists for plotting
159
237
x = list(time_data)
160
238
y_free = list(free_mem_data)
161
239
y_total = list(total_mem_data)
162
240
163
-
plt.cla() # Clear axis
241
+
plt.cla() # Clear axis
164
242
165
243
# Plot Total RAM
166
-
plt.plot(x, y_total, label='Total RAM (KB)', color='red', linestyle='--')
244
+
plt.plot(x, y_total, label="Total RAM (KB)", color="red", linestyle="--")
167
245
168
246
# Plot Free RAM
169
-
plt.plot(x, y_free, label='Free RAM (KB)', color='green', marker='o')
247
+
plt.plot(x, y_free, label="Free RAM (KB)", color="green", marker="o")
170
248
171
249
# Fill area under Free RAM
172
-
plt.fill_between(x, y_free, color='green', alpha=0.1)
250
+
plt.fill_between(x, y_free, color="green", alpha=0.1)
173
251
174
252
plt.title("ESP32 Memory Monitor")
175
253
plt.ylabel("Memory (KB)")
176
254
plt.xlabel("Time")
177
-
plt.legend(loc='upper left')
178
-
plt.grid(True, linestyle=':', alpha=0.6)
255
+
plt.legend(loc="upper left")
256
+
plt.grid(True, linestyle=":", alpha=0.6)
179
257
180
258
# Rotate date labels
181
-
plt.xticks(rotation=45, ha='right')
259
+
plt.xticks(rotation=45, ha="right")
182
260
plt.tight_layout()
183
261
184
-
def main():
262
+
return []
263
+
264
+
265
+
def main() -> None:
266
+
"""
267
+
Main entry point for the ESP32 monitor application.
268
+
Sets up argument parsing, starts serial monitoring thread, and initializes the memory graph.
269
+
"""
185
270
parser = argparse.ArgumentParser(description="ESP32 Monitor with Graph")
186
-
parser.add_argument("port", nargs="?", default="/dev/ttyACM0", help="Serial port")
187
-
parser.add_argument("--baud", type=int, default=115200, help="Baud rate")
271
+
if sys.platform.startswith("win"):
272
+
default_port = "COM8"
273
+
elif sys.platform.startswith("darwin"):
274
+
default_port = "/dev/cu.usbmodem101"
275
+
else:
276
+
default_port = "/dev/ttyACM0"
277
+
default_baudrate = 115200
278
+
parser.add_argument(
279
+
"port",
280
+
nargs="?",
281
+
default=default_port,
282
+
help=f"Serial port (default: {default_port})",
283
+
)
284
+
parser.add_argument(
285
+
"--baud",
286
+
type=int,
287
+
default=default_baudrate,
288
+
help=f"Baud rate (default: {default_baudrate})",
289
+
)
290
+
parser.add_argument(
291
+
"--filter",
292
+
type=str,
293
+
default="",
294
+
help="Only display lines containing this keyword (case-insensitive)",
295
+
)
296
+
parser.add_argument(
297
+
"--suppress",
298
+
type=str,
299
+
default="",
300
+
help="Suppress lines containing this keyword (case-insensitive)",
301
+
)
188
302
args = parser.parse_args()
189
303
190
304
# 1. Start the Serial Reader in a separate thread
191
305
# Daemon=True means this thread dies when the main program closes
192
-
t = threading.Thread(target=serial_worker, args=(args.port, args.baud), daemon=True)
306
+
myargs = vars(args) # Convert Namespace to dict for easier passing
307
+
t = threading.Thread(
308
+
target=serial_worker, args=(args.port, args.baud, myargs), daemon=True
309
+
)
193
310
t.start()
194
311
195
312
# 2. Set up the Graph (Main Thread)
196
313
try:
197
-
plt.style.use('light_background')
198
-
except:
314
+
import matplotlib.style as mplstyle # pylint: disable=import-outside-toplevel
315
+
default_styles = ("light_background", "ggplot", "seaborn", "dark_background", )
316
+
styles = list(mplstyle.available)
317
+
for default_style in default_styles:
318
+
if default_style in styles:
319
+
print(
320
+
f"\n{Fore.CYAN}--- Using Matplotlib style: {default_style} ---{Style.RESET_ALL}"
321
+
)
322
+
mplstyle.use(default_style)
323
+
break
324
+
except (AttributeError, ValueError):
199
325
pass
200
326
201
327
fig = plt.figure(figsize=(10, 6))
202
328
203
329
# Update graph every 1000ms
204
-
ani = animation.FuncAnimation(fig, update_graph, interval=1000)
330
+
_ = animation.FuncAnimation(
331
+
fig, update_graph, interval=1000, cache_frame_data=False
332
+
)
205
333
206
334
try:
207
-
print(f"{Fore.YELLOW}Starting Graph Window... (Close window to exit){Style.RESET_ALL}")
335
+
print(
336
+
f"{Fore.YELLOW}Starting Graph Window... (Close window to exit){Style.RESET_ALL}"
337
+
)
208
338
plt.show()
209
339
except KeyboardInterrupt:
210
340
print(f"\n{Fore.YELLOW}Exiting...{Style.RESET_ALL}")
211
-
plt.close('all') # Force close any lingering plot windows
341
+
plt.close("all") # Force close any lingering plot windows
342
+
212
343
213
344
if __name__ == "__main__":
214
345
main()