Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
perf script python: Add support for exporting to sqlite3
Add support for exporting to SQLite 3 the same data as the PostgreSQL
export.
Committer note:
Tested on RHEL 7.4 using the 1.2.2-4el python-pyside packages from EPEL.
Signed-off-by: Adrian Hunter <adrian.hunter@intel.com>
Tested-by: Arnaldo Carvalho de Melo <acme@redhat.com>
Link: http://lkml.kernel.org/r/1501749090-20357-4-git-send-email-adrian.hunter@intel.com
Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com>
authored by
Adrian Hunter
and committed by
Arnaldo Carvalho de Melo
564b9527
2295e9f8
+491
-3
+3
-3
tools/perf/Documentation/intel-pt.txt
+3
-3
tools/perf/Documentation/intel-pt.txt
···
104
104
in transaction, respectively.
105
105
106
106
While it is possible to create scripts to analyze the data, an alternative
107
-
approach is available to export the data to a postgresql database. Refer to
108
-
script export-to-postgresql.py for more details, and to script
109
-
call-graph-from-postgresql.py for an example of using the database.
107
+
approach is available to export the data to a sqlite or postgresql database.
108
+
Refer to script export-to-sqlite.py or export-to-postgresql.py for more details,
109
+
and to script call-graph-from-postgresql.py for an example of using the database.
110
110
111
111
There is also script intel-pt-events.py which provides an example of how to
112
112
unpack the raw data for power events and PTWRITE.
+8
tools/perf/scripts/python/bin/export-to-sqlite-record
+8
tools/perf/scripts/python/bin/export-to-sqlite-record
+29
tools/perf/scripts/python/bin/export-to-sqlite-report
+29
tools/perf/scripts/python/bin/export-to-sqlite-report
···
1
+
#!/bin/bash
2
+
# description: export perf data to a sqlite3 database
3
+
# args: [database name] [columns] [calls]
4
+
n_args=0
5
+
for i in "$@"
6
+
do
7
+
if expr match "$i" "-" > /dev/null ; then
8
+
break
9
+
fi
10
+
n_args=$(( $n_args + 1 ))
11
+
done
12
+
if [ "$n_args" -gt 3 ] ; then
13
+
echo "usage: export-to-sqlite-report [database name] [columns] [calls]"
14
+
exit
15
+
fi
16
+
if [ "$n_args" -gt 2 ] ; then
17
+
dbname=$1
18
+
columns=$2
19
+
calls=$3
20
+
shift 3
21
+
elif [ "$n_args" -gt 1 ] ; then
22
+
dbname=$1
23
+
columns=$2
24
+
shift 2
25
+
elif [ "$n_args" -gt 0 ] ; then
26
+
dbname=$1
27
+
shift
28
+
fi
29
+
perf script $@ -s "$PERF_EXEC_PATH"/scripts/python/export-to-sqlite.py $dbname $columns $calls
+451
tools/perf/scripts/python/export-to-sqlite.py
+451
tools/perf/scripts/python/export-to-sqlite.py
···
1
+
# export-to-sqlite.py: export perf data to a sqlite3 database
2
+
# Copyright (c) 2017, Intel Corporation.
3
+
#
4
+
# This program is free software; you can redistribute it and/or modify it
5
+
# under the terms and conditions of the GNU General Public License,
6
+
# version 2, as published by the Free Software Foundation.
7
+
#
8
+
# This program is distributed in the hope it will be useful, but WITHOUT
9
+
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10
+
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11
+
# more details.
12
+
13
+
import os
14
+
import sys
15
+
import struct
16
+
import datetime
17
+
18
+
# To use this script you will need to have installed package python-pyside which
19
+
# provides LGPL-licensed Python bindings for Qt. You will also need the package
20
+
# libqt4-sql-sqlite for Qt sqlite3 support.
21
+
#
22
+
# An example of using this script with Intel PT:
23
+
#
24
+
# $ perf record -e intel_pt//u ls
25
+
# $ perf script -s ~/libexec/perf-core/scripts/python/export-to-sqlite.py pt_example branches calls
26
+
# 2017-07-31 14:26:07.326913 Creating database...
27
+
# 2017-07-31 14:26:07.538097 Writing records...
28
+
# 2017-07-31 14:26:09.889292 Adding indexes
29
+
# 2017-07-31 14:26:09.958746 Done
30
+
#
31
+
# To browse the database, sqlite3 can be used e.g.
32
+
#
33
+
# $ sqlite3 pt_example
34
+
# sqlite> .header on
35
+
# sqlite> select * from samples_view where id < 10;
36
+
# sqlite> .mode column
37
+
# sqlite> select * from samples_view where id < 10;
38
+
# sqlite> .tables
39
+
# sqlite> .schema samples_view
40
+
# sqlite> .quit
41
+
#
42
+
# An example of using the database is provided by the script
43
+
# call-graph-from-sql.py. Refer to that script for details.
44
+
#
45
+
# The database structure is practically the same as created by the script
46
+
# export-to-postgresql.py. Refer to that script for details. A notable
47
+
# difference is the 'transaction' column of the 'samples' table which is
48
+
# renamed 'transaction_' in sqlite because 'transaction' is a reserved word.
49
+
50
+
from PySide.QtSql import *
51
+
52
+
sys.path.append(os.environ['PERF_EXEC_PATH'] + \
53
+
'/scripts/python/Perf-Trace-Util/lib/Perf/Trace')
54
+
55
+
# These perf imports are not used at present
56
+
#from perf_trace_context import *
57
+
#from Core import *
58
+
59
+
perf_db_export_mode = True
60
+
perf_db_export_calls = False
61
+
perf_db_export_callchains = False
62
+
63
+
def usage():
64
+
print >> sys.stderr, "Usage is: export-to-sqlite.py <database name> [<columns>] [<calls>] [<callchains>]"
65
+
print >> sys.stderr, "where: columns 'all' or 'branches'"
66
+
print >> sys.stderr, " calls 'calls' => create calls and call_paths table"
67
+
print >> sys.stderr, " callchains 'callchains' => create call_paths table"
68
+
raise Exception("Too few arguments")
69
+
70
+
if (len(sys.argv) < 2):
71
+
usage()
72
+
73
+
dbname = sys.argv[1]
74
+
75
+
if (len(sys.argv) >= 3):
76
+
columns = sys.argv[2]
77
+
else:
78
+
columns = "all"
79
+
80
+
if columns not in ("all", "branches"):
81
+
usage()
82
+
83
+
branches = (columns == "branches")
84
+
85
+
for i in range(3,len(sys.argv)):
86
+
if (sys.argv[i] == "calls"):
87
+
perf_db_export_calls = True
88
+
elif (sys.argv[i] == "callchains"):
89
+
perf_db_export_callchains = True
90
+
else:
91
+
usage()
92
+
93
+
def do_query(q, s):
94
+
if (q.exec_(s)):
95
+
return
96
+
raise Exception("Query failed: " + q.lastError().text())
97
+
98
+
def do_query_(q):
99
+
if (q.exec_()):
100
+
return
101
+
raise Exception("Query failed: " + q.lastError().text())
102
+
103
+
print datetime.datetime.today(), "Creating database..."
104
+
105
+
db_exists = False
106
+
try:
107
+
f = open(dbname)
108
+
f.close()
109
+
db_exists = True
110
+
except:
111
+
pass
112
+
113
+
if db_exists:
114
+
raise Exception(dbname + " already exists")
115
+
116
+
db = QSqlDatabase.addDatabase('QSQLITE')
117
+
db.setDatabaseName(dbname)
118
+
db.open()
119
+
120
+
query = QSqlQuery(db)
121
+
122
+
do_query(query, 'PRAGMA journal_mode = OFF')
123
+
do_query(query, 'BEGIN TRANSACTION')
124
+
125
+
do_query(query, 'CREATE TABLE selected_events ('
126
+
'id integer NOT NULL PRIMARY KEY,'
127
+
'name varchar(80))')
128
+
do_query(query, 'CREATE TABLE machines ('
129
+
'id integer NOT NULL PRIMARY KEY,'
130
+
'pid integer,'
131
+
'root_dir varchar(4096))')
132
+
do_query(query, 'CREATE TABLE threads ('
133
+
'id integer NOT NULL PRIMARY KEY,'
134
+
'machine_id bigint,'
135
+
'process_id bigint,'
136
+
'pid integer,'
137
+
'tid integer)')
138
+
do_query(query, 'CREATE TABLE comms ('
139
+
'id integer NOT NULL PRIMARY KEY,'
140
+
'comm varchar(16))')
141
+
do_query(query, 'CREATE TABLE comm_threads ('
142
+
'id integer NOT NULL PRIMARY KEY,'
143
+
'comm_id bigint,'
144
+
'thread_id bigint)')
145
+
do_query(query, 'CREATE TABLE dsos ('
146
+
'id integer NOT NULL PRIMARY KEY,'
147
+
'machine_id bigint,'
148
+
'short_name varchar(256),'
149
+
'long_name varchar(4096),'
150
+
'build_id varchar(64))')
151
+
do_query(query, 'CREATE TABLE symbols ('
152
+
'id integer NOT NULL PRIMARY KEY,'
153
+
'dso_id bigint,'
154
+
'sym_start bigint,'
155
+
'sym_end bigint,'
156
+
'binding integer,'
157
+
'name varchar(2048))')
158
+
do_query(query, 'CREATE TABLE branch_types ('
159
+
'id integer NOT NULL PRIMARY KEY,'
160
+
'name varchar(80))')
161
+
162
+
if branches:
163
+
do_query(query, 'CREATE TABLE samples ('
164
+
'id integer NOT NULL PRIMARY KEY,'
165
+
'evsel_id bigint,'
166
+
'machine_id bigint,'
167
+
'thread_id bigint,'
168
+
'comm_id bigint,'
169
+
'dso_id bigint,'
170
+
'symbol_id bigint,'
171
+
'sym_offset bigint,'
172
+
'ip bigint,'
173
+
'time bigint,'
174
+
'cpu integer,'
175
+
'to_dso_id bigint,'
176
+
'to_symbol_id bigint,'
177
+
'to_sym_offset bigint,'
178
+
'to_ip bigint,'
179
+
'branch_type integer,'
180
+
'in_tx boolean,'
181
+
'call_path_id bigint)')
182
+
else:
183
+
do_query(query, 'CREATE TABLE samples ('
184
+
'id integer NOT NULL PRIMARY KEY,'
185
+
'evsel_id bigint,'
186
+
'machine_id bigint,'
187
+
'thread_id bigint,'
188
+
'comm_id bigint,'
189
+
'dso_id bigint,'
190
+
'symbol_id bigint,'
191
+
'sym_offset bigint,'
192
+
'ip bigint,'
193
+
'time bigint,'
194
+
'cpu integer,'
195
+
'to_dso_id bigint,'
196
+
'to_symbol_id bigint,'
197
+
'to_sym_offset bigint,'
198
+
'to_ip bigint,'
199
+
'period bigint,'
200
+
'weight bigint,'
201
+
'transaction_ bigint,'
202
+
'data_src bigint,'
203
+
'branch_type integer,'
204
+
'in_tx boolean,'
205
+
'call_path_id bigint)')
206
+
207
+
if perf_db_export_calls or perf_db_export_callchains:
208
+
do_query(query, 'CREATE TABLE call_paths ('
209
+
'id integer NOT NULL PRIMARY KEY,'
210
+
'parent_id bigint,'
211
+
'symbol_id bigint,'
212
+
'ip bigint)')
213
+
if perf_db_export_calls:
214
+
do_query(query, 'CREATE TABLE calls ('
215
+
'id integer NOT NULL PRIMARY KEY,'
216
+
'thread_id bigint,'
217
+
'comm_id bigint,'
218
+
'call_path_id bigint,'
219
+
'call_time bigint,'
220
+
'return_time bigint,'
221
+
'branch_count bigint,'
222
+
'call_id bigint,'
223
+
'return_id bigint,'
224
+
'parent_call_path_id bigint,'
225
+
'flags integer)')
226
+
227
+
# printf was added to sqlite in version 3.8.3
228
+
sqlite_has_printf = False
229
+
try:
230
+
do_query(query, 'SELECT printf("") FROM machines')
231
+
sqlite_has_printf = True
232
+
except:
233
+
pass
234
+
235
+
def emit_to_hex(x):
236
+
if sqlite_has_printf:
237
+
return 'printf("%x", ' + x + ')'
238
+
else:
239
+
return x
240
+
241
+
do_query(query, 'CREATE VIEW machines_view AS '
242
+
'SELECT '
243
+
'id,'
244
+
'pid,'
245
+
'root_dir,'
246
+
'CASE WHEN id=0 THEN \'unknown\' WHEN pid=-1 THEN \'host\' ELSE \'guest\' END AS host_or_guest'
247
+
' FROM machines')
248
+
249
+
do_query(query, 'CREATE VIEW dsos_view AS '
250
+
'SELECT '
251
+
'id,'
252
+
'machine_id,'
253
+
'(SELECT host_or_guest FROM machines_view WHERE id = machine_id) AS host_or_guest,'
254
+
'short_name,'
255
+
'long_name,'
256
+
'build_id'
257
+
' FROM dsos')
258
+
259
+
do_query(query, 'CREATE VIEW symbols_view AS '
260
+
'SELECT '
261
+
'id,'
262
+
'name,'
263
+
'(SELECT short_name FROM dsos WHERE id=dso_id) AS dso,'
264
+
'dso_id,'
265
+
'sym_start,'
266
+
'sym_end,'
267
+
'CASE WHEN binding=0 THEN \'local\' WHEN binding=1 THEN \'global\' ELSE \'weak\' END AS binding'
268
+
' FROM symbols')
269
+
270
+
do_query(query, 'CREATE VIEW threads_view AS '
271
+
'SELECT '
272
+
'id,'
273
+
'machine_id,'
274
+
'(SELECT host_or_guest FROM machines_view WHERE id = machine_id) AS host_or_guest,'
275
+
'process_id,'
276
+
'pid,'
277
+
'tid'
278
+
' FROM threads')
279
+
280
+
do_query(query, 'CREATE VIEW comm_threads_view AS '
281
+
'SELECT '
282
+
'comm_id,'
283
+
'(SELECT comm FROM comms WHERE id = comm_id) AS command,'
284
+
'thread_id,'
285
+
'(SELECT pid FROM threads WHERE id = thread_id) AS pid,'
286
+
'(SELECT tid FROM threads WHERE id = thread_id) AS tid'
287
+
' FROM comm_threads')
288
+
289
+
if perf_db_export_calls or perf_db_export_callchains:
290
+
do_query(query, 'CREATE VIEW call_paths_view AS '
291
+
'SELECT '
292
+
'c.id,'
293
+
+ emit_to_hex('c.ip') + ' AS ip,'
294
+
'c.symbol_id,'
295
+
'(SELECT name FROM symbols WHERE id = c.symbol_id) AS symbol,'
296
+
'(SELECT dso_id FROM symbols WHERE id = c.symbol_id) AS dso_id,'
297
+
'(SELECT dso FROM symbols_view WHERE id = c.symbol_id) AS dso_short_name,'
298
+
'c.parent_id,'
299
+
+ emit_to_hex('p.ip') + ' AS parent_ip,'
300
+
'p.symbol_id AS parent_symbol_id,'
301
+
'(SELECT name FROM symbols WHERE id = p.symbol_id) AS parent_symbol,'
302
+
'(SELECT dso_id FROM symbols WHERE id = p.symbol_id) AS parent_dso_id,'
303
+
'(SELECT dso FROM symbols_view WHERE id = p.symbol_id) AS parent_dso_short_name'
304
+
' FROM call_paths c INNER JOIN call_paths p ON p.id = c.parent_id')
305
+
if perf_db_export_calls:
306
+
do_query(query, 'CREATE VIEW calls_view AS '
307
+
'SELECT '
308
+
'calls.id,'
309
+
'thread_id,'
310
+
'(SELECT pid FROM threads WHERE id = thread_id) AS pid,'
311
+
'(SELECT tid FROM threads WHERE id = thread_id) AS tid,'
312
+
'(SELECT comm FROM comms WHERE id = comm_id) AS command,'
313
+
'call_path_id,'
314
+
+ emit_to_hex('ip') + ' AS ip,'
315
+
'symbol_id,'
316
+
'(SELECT name FROM symbols WHERE id = symbol_id) AS symbol,'
317
+
'call_time,'
318
+
'return_time,'
319
+
'return_time - call_time AS elapsed_time,'
320
+
'branch_count,'
321
+
'call_id,'
322
+
'return_id,'
323
+
'CASE WHEN flags=1 THEN \'no call\' WHEN flags=2 THEN \'no return\' WHEN flags=3 THEN \'no call/return\' ELSE \'\' END AS flags,'
324
+
'parent_call_path_id'
325
+
' FROM calls INNER JOIN call_paths ON call_paths.id = call_path_id')
326
+
327
+
do_query(query, 'CREATE VIEW samples_view AS '
328
+
'SELECT '
329
+
'id,'
330
+
'time,'
331
+
'cpu,'
332
+
'(SELECT pid FROM threads WHERE id = thread_id) AS pid,'
333
+
'(SELECT tid FROM threads WHERE id = thread_id) AS tid,'
334
+
'(SELECT comm FROM comms WHERE id = comm_id) AS command,'
335
+
'(SELECT name FROM selected_events WHERE id = evsel_id) AS event,'
336
+
+ emit_to_hex('ip') + ' AS ip_hex,'
337
+
'(SELECT name FROM symbols WHERE id = symbol_id) AS symbol,'
338
+
'sym_offset,'
339
+
'(SELECT short_name FROM dsos WHERE id = dso_id) AS dso_short_name,'
340
+
+ emit_to_hex('to_ip') + ' AS to_ip_hex,'
341
+
'(SELECT name FROM symbols WHERE id = to_symbol_id) AS to_symbol,'
342
+
'to_sym_offset,'
343
+
'(SELECT short_name FROM dsos WHERE id = to_dso_id) AS to_dso_short_name,'
344
+
'(SELECT name FROM branch_types WHERE id = branch_type) AS branch_type_name,'
345
+
'in_tx'
346
+
' FROM samples')
347
+
348
+
do_query(query, 'END TRANSACTION')
349
+
350
+
evsel_query = QSqlQuery(db)
351
+
evsel_query.prepare("INSERT INTO selected_events VALUES (?, ?)")
352
+
machine_query = QSqlQuery(db)
353
+
machine_query.prepare("INSERT INTO machines VALUES (?, ?, ?)")
354
+
thread_query = QSqlQuery(db)
355
+
thread_query.prepare("INSERT INTO threads VALUES (?, ?, ?, ?, ?)")
356
+
comm_query = QSqlQuery(db)
357
+
comm_query.prepare("INSERT INTO comms VALUES (?, ?)")
358
+
comm_thread_query = QSqlQuery(db)
359
+
comm_thread_query.prepare("INSERT INTO comm_threads VALUES (?, ?, ?)")
360
+
dso_query = QSqlQuery(db)
361
+
dso_query.prepare("INSERT INTO dsos VALUES (?, ?, ?, ?, ?)")
362
+
symbol_query = QSqlQuery(db)
363
+
symbol_query.prepare("INSERT INTO symbols VALUES (?, ?, ?, ?, ?, ?)")
364
+
branch_type_query = QSqlQuery(db)
365
+
branch_type_query.prepare("INSERT INTO branch_types VALUES (?, ?)")
366
+
sample_query = QSqlQuery(db)
367
+
if branches:
368
+
sample_query.prepare("INSERT INTO samples VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
369
+
else:
370
+
sample_query.prepare("INSERT INTO samples VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
371
+
if perf_db_export_calls or perf_db_export_callchains:
372
+
call_path_query = QSqlQuery(db)
373
+
call_path_query.prepare("INSERT INTO call_paths VALUES (?, ?, ?, ?)")
374
+
if perf_db_export_calls:
375
+
call_query = QSqlQuery(db)
376
+
call_query.prepare("INSERT INTO calls VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
377
+
378
+
def trace_begin():
379
+
print datetime.datetime.today(), "Writing records..."
380
+
do_query(query, 'BEGIN TRANSACTION')
381
+
# id == 0 means unknown. It is easier to create records for them than replace the zeroes with NULLs
382
+
evsel_table(0, "unknown")
383
+
machine_table(0, 0, "unknown")
384
+
thread_table(0, 0, 0, -1, -1)
385
+
comm_table(0, "unknown")
386
+
dso_table(0, 0, "unknown", "unknown", "")
387
+
symbol_table(0, 0, 0, 0, 0, "unknown")
388
+
sample_table(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)
389
+
if perf_db_export_calls or perf_db_export_callchains:
390
+
call_path_table(0, 0, 0, 0)
391
+
392
+
unhandled_count = 0
393
+
394
+
def trace_end():
395
+
do_query(query, 'END TRANSACTION')
396
+
397
+
print datetime.datetime.today(), "Adding indexes"
398
+
if perf_db_export_calls:
399
+
do_query(query, 'CREATE INDEX pcpid_idx ON calls (parent_call_path_id)')
400
+
401
+
if (unhandled_count):
402
+
print datetime.datetime.today(), "Warning: ", unhandled_count, " unhandled events"
403
+
print datetime.datetime.today(), "Done"
404
+
405
+
def trace_unhandled(event_name, context, event_fields_dict):
406
+
global unhandled_count
407
+
unhandled_count += 1
408
+
409
+
def sched__sched_switch(*x):
410
+
pass
411
+
412
+
def bind_exec(q, n, x):
413
+
for xx in x[0:n]:
414
+
q.addBindValue(str(xx))
415
+
do_query_(q)
416
+
417
+
def evsel_table(*x):
418
+
bind_exec(evsel_query, 2, x)
419
+
420
+
def machine_table(*x):
421
+
bind_exec(machine_query, 3, x)
422
+
423
+
def thread_table(*x):
424
+
bind_exec(thread_query, 5, x)
425
+
426
+
def comm_table(*x):
427
+
bind_exec(comm_query, 2, x)
428
+
429
+
def comm_thread_table(*x):
430
+
bind_exec(comm_thread_query, 3, x)
431
+
432
+
def dso_table(*x):
433
+
bind_exec(dso_query, 5, x)
434
+
435
+
def symbol_table(*x):
436
+
bind_exec(symbol_query, 6, x)
437
+
438
+
def branch_type_table(*x):
439
+
bind_exec(branch_type_query, 2, x)
440
+
441
+
def sample_table(*x):
442
+
if branches:
443
+
bind_exec(sample_query, 18, x)
444
+
else:
445
+
bind_exec(sample_query, 22, x)
446
+
447
+
def call_path_table(*x):
448
+
bind_exec(call_path_query, 4, x)
449
+
450
+
def call_return_table(*x):
451
+
bind_exec(call_query, 11, x)