tools/perf/util/intel-pt.c

   1 /*
   2  * intel_pt.c: Intel Processor Trace support
   3  * Copyright (c) 2013-2015, Intel Corporation.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  *
  14  */
  15
  16 #include <inttypes.h>
  17 #include <stdio.h>
  18 #include <stdbool.h>
  19 #include <errno.h>
  20 #include <linux/kernel.h>
  21 #include <linux/types.h>
  22
  23 #include "../perf.h"
  24 #include "session.h"
  25 #include "machine.h"
  26 #include "memswap.h"
  27 #include "sort.h"
  28 #include "tool.h"
  29 #include "event.h"
  30 #include "evlist.h"
  31 #include "evsel.h"
  32 #include "map.h"
  33 #include "color.h"
  34 #include "util.h"
  35 #include "thread.h"
  36 #include "thread-stack.h"
  37 #include "symbol.h"
  38 #include "callchain.h"
  39 #include "dso.h"
  40 #include "debug.h"
  41 #include "auxtrace.h"
  42 #include "tsc.h"
  43 #include "intel-pt.h"
  44 #include "config.h"
  45
  46 #include "intel-pt-decoder/intel-pt-log.h"
  47 #include "intel-pt-decoder/intel-pt-decoder.h"
  48 #include "intel-pt-decoder/intel-pt-insn-decoder.h"
  49 #include "intel-pt-decoder/intel-pt-pkt-decoder.h"
  50
  51 #define MAX_TIMESTAMP (~0ULL)
  52
  53 struct intel_pt {
  54         struct auxtrace auxtrace;
  55         struct auxtrace_queues queues;
  56         struct auxtrace_heap heap;
  57         u32 auxtrace_type;
  58         struct perf_session *session;
  59         struct machine *machine;
  60         struct perf_evsel *switch_evsel;
  61         struct thread *unknown_thread;
  62         bool timeless_decoding;
  63         bool sampling_mode;
  64         bool snapshot_mode;
  65         bool per_cpu_mmaps;
  66         bool have_tsc;
  67         bool data_queued;
  68         bool est_tsc;
  69         bool sync_switch;
  70         bool mispred_all;
  71         int have_sched_switch;
  72         u32 pmu_type;
  73         u64 kernel_start;
  74         u64 switch_ip;
  75         u64 ptss_ip;
  76
  77         struct perf_tsc_conversion tc;
  78         bool cap_user_time_zero;
  79
  80         struct itrace_synth_opts synth_opts;
  81
  82         bool sample_instructions;
  83         u64 instructions_sample_type;
  84         u64 instructions_id;
  85
  86         bool sample_branches;
  87         u32 branches_filter;
  88         u64 branches_sample_type;
  89         u64 branches_id;
  90
  91         bool sample_transactions;
  92         u64 transactions_sample_type;
  93         u64 transactions_id;
  94
  95         bool sample_ptwrites;
  96         u64 ptwrites_sample_type;
  97         u64 ptwrites_id;
  98
  99         bool sample_pwr_events;
 100         u64 pwr_events_sample_type;
 101         u64 mwait_id;
 102         u64 pwre_id;
 103         u64 exstop_id;
 104         u64 pwrx_id;
 105         u64 cbr_id;
 106
 107         bool synth_needs_swap;
 108
 109         u64 tsc_bit;
 110         u64 mtc_bit;
 111         u64 mtc_freq_bits;
 112         u32 tsc_ctc_ratio_n;
 113         u32 tsc_ctc_ratio_d;
 114         u64 cyc_bit;
 115         u64 noretcomp_bit;
 116         unsigned max_non_turbo_ratio;
 117         unsigned cbr2khz;
 118
 119         unsigned long num_events;
 120
 121         char *filter;
 122         struct addr_filters filts;
 123 };
 124
 125 enum switch_state {
 126         INTEL_PT_SS_NOT_TRACING,
 127         INTEL_PT_SS_UNKNOWN,
 128         INTEL_PT_SS_TRACING,
 129         INTEL_PT_SS_EXPECTING_SWITCH_EVENT,
 130         INTEL_PT_SS_EXPECTING_SWITCH_IP,
 131 };
 132
 133 struct intel_pt_queue {
 134         struct intel_pt *pt;
 135         unsigned int queue_nr;
 136         struct auxtrace_buffer *buffer;
 137         void *decoder;
 138         const struct intel_pt_state *state;
 139         struct ip_callchain *chain;
 140         struct branch_stack *last_branch;
 141         struct branch_stack *last_branch_rb;
 142         size_t last_branch_pos;
 143         union perf_event *event_buf;
 144         bool on_heap;
 145         bool stop;
 146         bool step_through_buffers;
 147         bool use_buffer_pid_tid;
 148         pid_t pid, tid;
 149         int cpu;
 150         int switch_state;
 151         pid_t next_tid;
 152         struct thread *thread;
 153         bool exclude_kernel;
 154         bool have_sample;
 155         u64 time;
 156         u64 timestamp;
 157         u32 flags;
 158         u16 insn_len;
 159         u64 last_insn_cnt;
 160         char insn[INTEL_PT_INSN_BUF_SZ];
 161 };
 162
 163 static void intel_pt_dump(struct intel_pt *pt __maybe_unused,
 164                           unsigned char *buf, size_t len)
 165 {
 166         struct intel_pt_pkt packet;
 167         size_t pos = 0;
 168         int ret, pkt_len, i;
 169         char desc[INTEL_PT_PKT_DESC_MAX];
 170         const char *color = PERF_COLOR_BLUE;
 171
 172         color_fprintf(stdout, color,
 173                       ". ... Intel Processor Trace data: size %zu bytes\n",
 174                       len);
 175
 176         while (len) {
 177                 ret = intel_pt_get_packet(buf, len, &packet);
 178                 if (ret > 0)
 179                         pkt_len = ret;
 180                 else
 181                         pkt_len = 1;
 182                 printf(".");
 183                 color_fprintf(stdout, color, "  %08x: ", pos);
 184                 for (i = 0; i < pkt_len; i++)
 185                         color_fprintf(stdout, color, " %02x", buf[i]);
 186                 for (; i < 16; i++)
 187                         color_fprintf(stdout, color, "   ");
 188                 if (ret > 0) {
 189                         ret = intel_pt_pkt_desc(&packet, desc,
 190                                                 INTEL_PT_PKT_DESC_MAX);
 191                         if (ret > 0)
 192                                 color_fprintf(stdout, color, " %s\n", desc);
 193                 } else {
 194                         color_fprintf(stdout, color, " Bad packet!\n");
 195                 }
 196                 pos += pkt_len;
 197                 buf += pkt_len;
 198                 len -= pkt_len;
 199         }
 200 }
 201
 202 static void intel_pt_dump_event(struct intel_pt *pt, unsigned char *buf,
 203                                 size_t len)
 204 {
 205         printf(".\n");
 206         intel_pt_dump(pt, buf, len);
 207 }
 208
 209 static int intel_pt_do_fix_overlap(struct intel_pt *pt, struct auxtrace_buffer *a,
 210                                    struct auxtrace_buffer *b)
 211 {
 212         void *start;
 213
 214         start = intel_pt_find_overlap(a->data, a->size, b->data, b->size,
 215                                       pt->have_tsc);
 216         if (!start)
 217                 return -EINVAL;
 218         b->use_size = b->data + b->size - start;
 219         b->use_data = start;
 220         return 0;
 221 }
 222
 223 static void intel_pt_use_buffer_pid_tid(struct intel_pt_queue *ptq,
 224                                         struct auxtrace_queue *queue,
 225                                         struct auxtrace_buffer *buffer)
 226 {
 227         if (queue->cpu == -1 && buffer->cpu != -1)
 228                 ptq->cpu = buffer->cpu;
 229
 230         ptq->pid = buffer->pid;
 231         ptq->tid = buffer->tid;
 232
 233         intel_pt_log("queue %u cpu %d pid %d tid %d\n",
 234                      ptq->queue_nr, ptq->cpu, ptq->pid, ptq->tid);
 235
 236         thread__zput(ptq->thread);
 237
 238         if (ptq->tid != -1) {
 239                 if (ptq->pid != -1)
 240                         ptq->thread = machine__findnew_thread(ptq->pt->machine,
 241                                                               ptq->pid,
 242                                                               ptq->tid);
 243                 else
 244                         ptq->thread = machine__find_thread(ptq->pt->machine, -1,
 245                                                            ptq->tid);
 246         }
 247 }
 248
 249 /* This function assumes data is processed sequentially only */
 250 static int intel_pt_get_trace(struct intel_pt_buffer *b, void *data)
 251 {
 252         struct intel_pt_queue *ptq = data;
 253         struct auxtrace_buffer *buffer = ptq->buffer, *old_buffer = buffer;
 254         struct auxtrace_queue *queue;
 255
 256         if (ptq->stop) {
 257                 b->len = 0;
 258                 return 0;
 259         }
 260
 261         queue = &ptq->pt->queues.queue_array[ptq->queue_nr];
 262 next:
 263         buffer = auxtrace_buffer__next(queue, buffer);
 264         if (!buffer) {
 265                 if (old_buffer)
 266                         auxtrace_buffer__drop_data(old_buffer);
 267                 b->len = 0;
 268                 return 0;
 269         }
 270
 271         ptq->buffer = buffer;
 272
 273         if (!buffer->data) {
 274                 int fd = perf_data_file__fd(ptq->pt->session->file);
 275
 276                 buffer->data = auxtrace_buffer__get_data(buffer, fd);
 277                 if (!buffer->data)
 278                         return -ENOMEM;
 279         }
 280
 281         if (ptq->pt->snapshot_mode && !buffer->consecutive && old_buffer &&
 282             intel_pt_do_fix_overlap(ptq->pt, old_buffer, buffer))
 283                 return -ENOMEM;
 284
 285         if (buffer->use_data) {
 286                 b->len = buffer->use_size;
 287                 b->buf = buffer->use_data;
 288         } else {
 289                 b->len = buffer->size;
 290                 b->buf = buffer->data;
 291         }
 292         b->ref_timestamp = buffer->reference;
 293
 294         /*
 295          * If in snapshot mode and the buffer has no usable data, get next
 296          * buffer and again check overlap against old_buffer.
 297          */
 298         if (ptq->pt->snapshot_mode && !b->len)
 299                 goto next;
 300
 301         if (old_buffer)
 302                 auxtrace_buffer__drop_data(old_buffer);
 303
 304         if (!old_buffer || ptq->pt->sampling_mode || (ptq->pt->snapshot_mode &&
 305                                                       !buffer->consecutive)) {
 306                 b->consecutive = false;
 307                 b->trace_nr = buffer->buffer_nr + 1;
 308         } else {
 309                 b->consecutive = true;
 310         }
 311
 312         if (ptq->use_buffer_pid_tid && (ptq->pid != buffer->pid ||
 313                                         ptq->tid != buffer->tid))
 314                 intel_pt_use_buffer_pid_tid(ptq, queue, buffer);
 315
 316         if (ptq->step_through_buffers)
 317                 ptq->stop = true;
 318
 319         if (!b->len)
 320                 return intel_pt_get_trace(b, data);
 321
 322         return 0;
 323 }
 324
 325 struct intel_pt_cache_entry {
 326         struct auxtrace_cache_entry     entry;
 327         u64                             insn_cnt;
 328         u64                             byte_cnt;
 329         enum intel_pt_insn_op           op;
 330         enum intel_pt_insn_branch       branch;
 331         int                             length;
 332         int32_t                         rel;
 333         char                            insn[INTEL_PT_INSN_BUF_SZ];
 334 };
 335
 336 static int intel_pt_config_div(const char *var, const char *value, void *data)
 337 {
 338         int *d = data;
 339         long val;
 340
 341         if (!strcmp(var, "intel-pt.cache-divisor")) {
 342                 val = strtol(value, NULL, 0);
 343                 if (val > 0 && val <= INT_MAX)
 344                         *d = val;
 345         }
 346
 347         return 0;
 348 }
 349
 350 static int intel_pt_cache_divisor(void)
 351 {
 352         static int d;
 353
 354         if (d)
 355                 return d;
 356
 357         perf_config(intel_pt_config_div, &d);
 358
 359         if (!d)
 360                 d = 64;
 361
 362         return d;
 363 }
 364
 365 static unsigned int intel_pt_cache_size(struct dso *dso,
 366                                         struct machine *machine)
 367 {
 368         off_t size;
 369
 370         size = dso__data_size(dso, machine);
 371         size /= intel_pt_cache_divisor();
 372         if (size < 1000)
 373                 return 10;
 374         if (size > (1 << 21))
 375                 return 21;
 376         return 32 - __builtin_clz(size);
 377 }
 378
 379 static struct auxtrace_cache *intel_pt_cache(struct dso *dso,
 380                                              struct machine *machine)
 381 {
 382         struct auxtrace_cache *c;
 383         unsigned int bits;
 384
 385         if (dso->auxtrace_cache)
 386                 return dso->auxtrace_cache;
 387
 388         bits = intel_pt_cache_size(dso, machine);
 389
 390         /* Ignoring cache creation failure */
 391         c = auxtrace_cache__new(bits, sizeof(struct intel_pt_cache_entry), 200);
 392
 393         dso->auxtrace_cache = c;
 394
 395         return c;
 396 }
 397
 398 static int intel_pt_cache_add(struct dso *dso, struct machine *machine,
 399                               u64 offset, u64 insn_cnt, u64 byte_cnt,
 400                               struct intel_pt_insn *intel_pt_insn)
 401 {
 402         struct auxtrace_cache *c = intel_pt_cache(dso, machine);
 403         struct intel_pt_cache_entry *e;
 404         int err;
 405
 406         if (!c)
 407                 return -ENOMEM;
 408
 409         e = auxtrace_cache__alloc_entry(c);
 410         if (!e)
 411                 return -ENOMEM;
 412
 413         e->insn_cnt = insn_cnt;
 414         e->byte_cnt = byte_cnt;
 415         e->op = intel_pt_insn->op;
 416         e->branch = intel_pt_insn->branch;
 417         e->length = intel_pt_insn->length;
 418         e->rel = intel_pt_insn->rel;
 419         memcpy(e->insn, intel_pt_insn->buf, INTEL_PT_INSN_BUF_SZ);
 420
 421         err = auxtrace_cache__add(c, offset, &e->entry);
 422         if (err)
 423                 auxtrace_cache__free_entry(c, e);
 424
 425         return err;
 426 }
 427
 428 static struct intel_pt_cache_entry *
 429 intel_pt_cache_lookup(struct dso *dso, struct machine *machine, u64 offset)
 430 {
 431         struct auxtrace_cache *c = intel_pt_cache(dso, machine);
 432
 433         if (!c)
 434                 return NULL;
 435
 436         return auxtrace_cache__lookup(dso->auxtrace_cache, offset);
 437 }
 438
 439 static int intel_pt_walk_next_insn(struct intel_pt_insn *intel_pt_insn,
 440                                    uint64_t *insn_cnt_ptr, uint64_t *ip,
 441                                    uint64_t to_ip, uint64_t max_insn_cnt,
 442                                    void *data)
 443 {
 444         struct intel_pt_queue *ptq = data;
 445         struct machine *machine = ptq->pt->machine;
 446         struct thread *thread;
 447         struct addr_location al;
 448         unsigned char buf[INTEL_PT_INSN_BUF_SZ];
 449         ssize_t len;
 450         int x86_64;
 451         u8 cpumode;
 452         u64 offset, start_offset, start_ip;
 453         u64 insn_cnt = 0;
 454         bool one_map = true;
 455
 456         intel_pt_insn->length = 0;
 457
 458         if (to_ip && *ip == to_ip)
 459                 goto out_no_cache;
 460
 461         if (*ip >= ptq->pt->kernel_start)
 462                 cpumode = PERF_RECORD_MISC_KERNEL;
 463         else
 464                 cpumode = PERF_RECORD_MISC_USER;
 465
 466         thread = ptq->thread;
 467         if (!thread) {
 468                 if (cpumode != PERF_RECORD_MISC_KERNEL)
 469                         return -EINVAL;
 470                 thread = ptq->pt->unknown_thread;
 471         }
 472
 473         while (1) {
 474                 thread__find_addr_map(thread, cpumode, MAP__FUNCTION, *ip, &al);
 475                 if (!al.map || !al.map->dso)
 476                         return -EINVAL;
 477
 478                 if (al.map->dso->data.status == DSO_DATA_STATUS_ERROR &&
 479                     dso__data_status_seen(al.map->dso,
 480                                           DSO_DATA_STATUS_SEEN_ITRACE))
 481                         return -ENOENT;
 482
 483                 offset = al.map->map_ip(al.map, *ip);
 484
 485                 if (!to_ip && one_map) {
 486                         struct intel_pt_cache_entry *e;
 487
 488                         e = intel_pt_cache_lookup(al.map->dso, machine, offset);
 489                         if (e &&
 490                             (!max_insn_cnt || e->insn_cnt <= max_insn_cnt)) {
 491                                 *insn_cnt_ptr = e->insn_cnt;
 492                                 *ip += e->byte_cnt;
 493                                 intel_pt_insn->op = e->op;
 494                                 intel_pt_insn->branch = e->branch;
 495                                 intel_pt_insn->length = e->length;
 496                                 intel_pt_insn->rel = e->rel;
 497                                 memcpy(intel_pt_insn->buf, e->insn,
 498                                        INTEL_PT_INSN_BUF_SZ);
 499                                 intel_pt_log_insn_no_data(intel_pt_insn, *ip);
 500                                 return 0;
 501                         }
 502                 }
 503
 504                 start_offset = offset;
 505                 start_ip = *ip;
 506
 507                 /* Load maps to ensure dso->is_64_bit has been updated */
 508                 map__load(al.map);
 509
 510                 x86_64 = al.map->dso->is_64_bit;
 511
 512                 while (1) {
 513                         len = dso__data_read_offset(al.map->dso, machine,
 514                                                     offset, buf,
 515                                                     INTEL_PT_INSN_BUF_SZ);
 516                         if (len <= 0)
 517                                 return -EINVAL;
 518
 519                         if (intel_pt_get_insn(buf, len, x86_64, intel_pt_insn))
 520                                 return -EINVAL;
 521
 522                         intel_pt_log_insn(intel_pt_insn, *ip);
 523
 524                         insn_cnt += 1;
 525
 526                         if (intel_pt_insn->branch != INTEL_PT_BR_NO_BRANCH)
 527                                 goto out;
 528
 529                         if (max_insn_cnt && insn_cnt >= max_insn_cnt)
 530                                 goto out_no_cache;
 531
 532                         *ip += intel_pt_insn->length;
 533
 534                         if (to_ip && *ip == to_ip)
 535                                 goto out_no_cache;
 536
 537                         if (*ip >= al.map->end)
 538                                 break;
 539
 540                         offset += intel_pt_insn->length;
 541                 }
 542                 one_map = false;
 543         }
 544 out:
 545         *insn_cnt_ptr = insn_cnt;
 546
 547         if (!one_map)
 548                 goto out_no_cache;
 549
 550         /*
 551          * Didn't lookup in the 'to_ip' case, so do it now to prevent duplicate
 552          * entries.
 553          */
 554         if (to_ip) {
 555                 struct intel_pt_cache_entry *e;
 556
 557                 e = intel_pt_cache_lookup(al.map->dso, machine, start_offset);
 558                 if (e)
 559                         return 0;
 560         }
 561
 562         /* Ignore cache errors */
 563         intel_pt_cache_add(al.map->dso, machine, start_offset, insn_cnt,
 564                            *ip - start_ip, intel_pt_insn);
 565
 566         return 0;
 567
 568 out_no_cache:
 569         *insn_cnt_ptr = insn_cnt;
 570         return 0;
 571 }
 572
 573 static bool intel_pt_match_pgd_ip(struct intel_pt *pt, uint64_t ip,
 574                                   uint64_t offset, const char *filename)
 575 {
 576         struct addr_filter *filt;
 577         bool have_filter   = false;
 578         bool hit_tracestop = false;
 579         bool hit_filter    = false;
 580
 581         list_for_each_entry(filt, &pt->filts.head, list) {
 582                 if (filt->start)
 583                         have_filter = true;
 584
 585                 if ((filename && !filt->filename) ||
 586                     (!filename && filt->filename) ||
 587                     (filename && strcmp(filename, filt->filename)))
 588                         continue;
 589
 590                 if (!(offset >= filt->addr && offset < filt->addr + filt->size))
 591                         continue;
 592
 593                 intel_pt_log("TIP.PGD ip %#"PRIx64" offset %#"PRIx64" in %s hit filter: %s offset %#"PRIx64" size %#"PRIx64"\n",
 594                              ip, offset, filename ? filename : "[kernel]",
 595                              filt->start ? "filter" : "stop",
 596                              filt->addr, filt->size);
 597
 598                 if (filt->start)
 599                         hit_filter = true;
 600                 else
 601                         hit_tracestop = true;
 602         }
 603
 604         if (!hit_tracestop && !hit_filter)
 605                 intel_pt_log("TIP.PGD ip %#"PRIx64" offset %#"PRIx64" in %s is not in a filter region\n",
 606                              ip, offset, filename ? filename : "[kernel]");
 607
 608         return hit_tracestop || (have_filter && !hit_filter);
 609 }
 610
 611 static int __intel_pt_pgd_ip(uint64_t ip, void *data)
 612 {
 613         struct intel_pt_queue *ptq = data;
 614         struct thread *thread;
 615         struct addr_location al;
 616         u8 cpumode;
 617         u64 offset;
 618
 619         if (ip >= ptq->pt->kernel_start)
 620                 return intel_pt_match_pgd_ip(ptq->pt, ip, ip, NULL);
 621
 622         cpumode = PERF_RECORD_MISC_USER;
 623
 624         thread = ptq->thread;
 625         if (!thread)
 626                 return -EINVAL;
 627
 628         thread__find_addr_map(thread, cpumode, MAP__FUNCTION, ip, &al);
 629         if (!al.map || !al.map->dso)
 630                 return -EINVAL;
 631
 632         offset = al.map->map_ip(al.map, ip);
 633
 634         return intel_pt_match_pgd_ip(ptq->pt, ip, offset,
 635                                      al.map->dso->long_name);
 636 }
 637
 638 static bool intel_pt_pgd_ip(uint64_t ip, void *data)
 639 {
 640         return __intel_pt_pgd_ip(ip, data) > 0;
 641 }
 642
 643 static bool intel_pt_get_config(struct intel_pt *pt,
 644                                 struct perf_event_attr *attr, u64 *config)
 645 {
 646         if (attr->type == pt->pmu_type) {
 647                 if (config)
 648                         *config = attr->config;
 649                 return true;
 650         }
 651
 652         return false;
 653 }
 654
 655 static bool intel_pt_exclude_kernel(struct intel_pt *pt)
 656 {
 657         struct perf_evsel *evsel;
 658
 659         evlist__for_each_entry(pt->session->evlist, evsel) {
 660                 if (intel_pt_get_config(pt, &evsel->attr, NULL) &&
 661                     !evsel->attr.exclude_kernel)
 662                         return false;
 663         }
 664         return true;
 665 }
 666
 667 static bool intel_pt_return_compression(struct intel_pt *pt)
 668 {
 669         struct perf_evsel *evsel;
 670         u64 config;
 671
 672         if (!pt->noretcomp_bit)
 673                 return true;
 674
 675         evlist__for_each_entry(pt->session->evlist, evsel) {
 676                 if (intel_pt_get_config(pt, &evsel->attr, &config) &&
 677                     (config & pt->noretcomp_bit))
 678                         return false;
 679         }
 680         return true;
 681 }
 682
 683 static bool intel_pt_branch_enable(struct intel_pt *pt)
 684 {
 685         struct perf_evsel *evsel;
 686         u64 config;
 687
 688         evlist__for_each_entry(pt->session->evlist, evsel) {
 689                 if (intel_pt_get_config(pt, &evsel->attr, &config) &&
 690                     (config & 1) && !(config & 0x2000))
 691                         return false;
 692         }
 693         return true;
 694 }
 695
 696 static unsigned int intel_pt_mtc_period(struct intel_pt *pt)
 697 {
 698         struct perf_evsel *evsel;
 699         unsigned int shift;
 700         u64 config;
 701
 702         if (!pt->mtc_freq_bits)
 703                 return 0;
 704
 705         for (shift = 0, config = pt->mtc_freq_bits; !(config & 1); shift++)
 706                 config >>= 1;
 707
 708         evlist__for_each_entry(pt->session->evlist, evsel) {
 709                 if (intel_pt_get_config(pt, &evsel->attr, &config))
 710                         return (config & pt->mtc_freq_bits) >> shift;
 711         }
 712         return 0;
 713 }
 714
 715 static bool intel_pt_timeless_decoding(struct intel_pt *pt)
 716 {
 717         struct perf_evsel *evsel;
 718         bool timeless_decoding = true;
 719         u64 config;
 720
 721         if (!pt->tsc_bit || !pt->cap_user_time_zero)
 722                 return true;
 723
 724         evlist__for_each_entry(pt->session->evlist, evsel) {
 725                 if (!(evsel->attr.sample_type & PERF_SAMPLE_TIME))
 726                         return true;
 727                 if (intel_pt_get_config(pt, &evsel->attr, &config)) {
 728                         if (config & pt->tsc_bit)
 729                                 timeless_decoding = false;
 730                         else
 731                                 return true;
 732                 }
 733         }
 734         return timeless_decoding;
 735 }
 736
 737 static bool intel_pt_tracing_kernel(struct intel_pt *pt)
 738 {
 739         struct perf_evsel *evsel;
 740
 741         evlist__for_each_entry(pt->session->evlist, evsel) {
 742                 if (intel_pt_get_config(pt, &evsel->attr, NULL) &&
 743                     !evsel->attr.exclude_kernel)
 744                         return true;
 745         }
 746         return false;
 747 }
 748
 749 static bool intel_pt_have_tsc(struct intel_pt *pt)
 750 {
 751         struct perf_evsel *evsel;
 752         bool have_tsc = false;
 753         u64 config;
 754
 755         if (!pt->tsc_bit)
 756                 return false;
 757
 758         evlist__for_each_entry(pt->session->evlist, evsel) {
 759                 if (intel_pt_get_config(pt, &evsel->attr, &config)) {
 760                         if (config & pt->tsc_bit)
 761                                 have_tsc = true;
 762                         else
 763                                 return false;
 764                 }
 765         }
 766         return have_tsc;
 767 }
 768
 769 static u64 intel_pt_ns_to_ticks(const struct intel_pt *pt, u64 ns)
 770 {
 771         u64 quot, rem;
 772
 773         quot = ns / pt->tc.time_mult;
 774         rem  = ns % pt->tc.time_mult;
 775         return (quot << pt->tc.time_shift) + (rem << pt->tc.time_shift) /
 776                 pt->tc.time_mult;
 777 }
 778
 779 static struct intel_pt_queue *intel_pt_alloc_queue(struct intel_pt *pt,
 780                                                    unsigned int queue_nr)
 781 {
 782         struct intel_pt_params params = { .get_trace = 0, };
 783         struct intel_pt_queue *ptq;
 784
 785         ptq = zalloc(sizeof(struct intel_pt_queue));
 786         if (!ptq)
 787                 return NULL;
 788
 789         if (pt->synth_opts.callchain) {
 790                 size_t sz = sizeof(struct ip_callchain);
 791
 792                 sz += pt->synth_opts.callchain_sz * sizeof(u64);
 793                 ptq->chain = zalloc(sz);
 794                 if (!ptq->chain)
 795                         goto out_free;
 796         }
 797
 798         if (pt->synth_opts.last_branch) {
 799                 size_t sz = sizeof(struct branch_stack);
 800
 801                 sz += pt->synth_opts.last_branch_sz *
 802                       sizeof(struct branch_entry);
 803                 ptq->last_branch = zalloc(sz);
 804                 if (!ptq->last_branch)
 805                         goto out_free;
 806                 ptq->last_branch_rb = zalloc(sz);
 807                 if (!ptq->last_branch_rb)
 808                         goto out_free;
 809         }
 810
 811         ptq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE);
 812         if (!ptq->event_buf)
 813                 goto out_free;
 814
 815         ptq->pt = pt;
 816         ptq->queue_nr = queue_nr;
 817         ptq->exclude_kernel = intel_pt_exclude_kernel(pt);
 818         ptq->pid = -1;
 819         ptq->tid = -1;
 820         ptq->cpu = -1;
 821         ptq->next_tid = -1;
 822
 823         params.get_trace = intel_pt_get_trace;
 824         params.walk_insn = intel_pt_walk_next_insn;
 825         params.data = ptq;
 826         params.return_compression = intel_pt_return_compression(pt);
 827         params.branch_enable = intel_pt_branch_enable(pt);
 828         params.max_non_turbo_ratio = pt->max_non_turbo_ratio;
 829         params.mtc_period = intel_pt_mtc_period(pt);
 830         params.tsc_ctc_ratio_n = pt->tsc_ctc_ratio_n;
 831         params.tsc_ctc_ratio_d = pt->tsc_ctc_ratio_d;
 832
 833         if (pt->filts.cnt > 0)
 834                 params.pgd_ip = intel_pt_pgd_ip;
 835
 836         if (pt->synth_opts.instructions) {
 837                 if (pt->synth_opts.period) {
 838                         switch (pt->synth_opts.period_type) {
 839                         case PERF_ITRACE_PERIOD_INSTRUCTIONS:
 840                                 params.period_type =
 841                                                 INTEL_PT_PERIOD_INSTRUCTIONS;
 842                                 params.period = pt->synth_opts.period;
 843                                 break;
 844                         case PERF_ITRACE_PERIOD_TICKS:
 845                                 params.period_type = INTEL_PT_PERIOD_TICKS;
 846                                 params.period = pt->synth_opts.period;
 847                                 break;
 848                         case PERF_ITRACE_PERIOD_NANOSECS:
 849                                 params.period_type = INTEL_PT_PERIOD_TICKS;
 850                                 params.period = intel_pt_ns_to_ticks(pt,
 851                                                         pt->synth_opts.period);
 852                                 break;
 853                         default:
 854                                 break;
 855                         }
 856                 }
 857
 858                 if (!params.period) {
 859                         params.period_type = INTEL_PT_PERIOD_INSTRUCTIONS;
 860                         params.period = 1;
 861                 }
 862         }
 863
 864         ptq->decoder = intel_pt_decoder_new(&params);
 865         if (!ptq->decoder)
 866                 goto out_free;
 867
 868         return ptq;
 869
 870 out_free:
 871         zfree(&ptq->event_buf);
 872         zfree(&ptq->last_branch);
 873         zfree(&ptq->last_branch_rb);
 874         zfree(&ptq->chain);
 875         free(ptq);
 876         return NULL;
 877 }
 878
 879 static void intel_pt_free_queue(void *priv)
 880 {
 881         struct intel_pt_queue *ptq = priv;
 882
 883         if (!ptq)
 884                 return;
 885         thread__zput(ptq->thread);
 886         intel_pt_decoder_free(ptq->decoder);
 887         zfree(&ptq->event_buf);
 888         zfree(&ptq->last_branch);
 889         zfree(&ptq->last_branch_rb);
 890         zfree(&ptq->chain);
 891         free(ptq);
 892 }
 893
 894 static void intel_pt_set_pid_tid_cpu(struct intel_pt *pt,
 895                                      struct auxtrace_queue *queue)
 896 {
 897         struct intel_pt_queue *ptq = queue->priv;
 898
 899         if (queue->tid == -1 || pt->have_sched_switch) {
 900                 ptq->tid = machine__get_current_tid(pt->machine, ptq->cpu);
 901                 thread__zput(ptq->thread);
 902         }
 903
 904         if (!ptq->thread && ptq->tid != -1)
 905                 ptq->thread = machine__find_thread(pt->machine, -1, ptq->tid);
 906
 907         if (ptq->thread) {
 908                 ptq->pid = ptq->thread->pid_;
 909                 if (queue->cpu == -1)
 910                         ptq->cpu = ptq->thread->cpu;
 911         }
 912 }
 913
 914 static void intel_pt_sample_flags(struct intel_pt_queue *ptq)
 915 {
 916         if (ptq->state->flags & INTEL_PT_ABORT_TX) {
 917                 ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_TX_ABORT;
 918         } else if (ptq->state->flags & INTEL_PT_ASYNC) {
 919                 if (ptq->state->to_ip)
 920                         ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL |
 921                                      PERF_IP_FLAG_ASYNC |
 922                                      PERF_IP_FLAG_INTERRUPT;
 923                 else
 924                         ptq->flags = PERF_IP_FLAG_BRANCH |
 925                                      PERF_IP_FLAG_TRACE_END;
 926                 ptq->insn_len = 0;
 927         } else {
 928                 if (ptq->state->from_ip)
 929                         ptq->flags = intel_pt_insn_type(ptq->state->insn_op);
 930                 else
 931                         ptq->flags = PERF_IP_FLAG_BRANCH |
 932                                      PERF_IP_FLAG_TRACE_BEGIN;
 933                 if (ptq->state->flags & INTEL_PT_IN_TX)
 934                         ptq->flags |= PERF_IP_FLAG_IN_TX;
 935                 ptq->insn_len = ptq->state->insn_len;
 936                 memcpy(ptq->insn, ptq->state->insn, INTEL_PT_INSN_BUF_SZ);
 937         }
 938 }
 939
 940 static int intel_pt_setup_queue(struct intel_pt *pt,
 941                                 struct auxtrace_queue *queue,
 942                                 unsigned int queue_nr)
 943 {
 944         struct intel_pt_queue *ptq = queue->priv;
 945
 946         if (list_empty(&queue->head))
 947                 return 0;
 948
 949         if (!ptq) {
 950                 ptq = intel_pt_alloc_queue(pt, queue_nr);
 951                 if (!ptq)
 952                         return -ENOMEM;
 953                 queue->priv = ptq;
 954
 955                 if (queue->cpu != -1)
 956                         ptq->cpu = queue->cpu;
 957                 ptq->tid = queue->tid;
 958
 959                 if (pt->sampling_mode) {
 960                         if (pt->timeless_decoding)
 961                                 ptq->step_through_buffers = true;
 962                         if (pt->timeless_decoding || !pt->have_sched_switch)
 963                                 ptq->use_buffer_pid_tid = true;
 964                 }
 965         }
 966
 967         if (!ptq->on_heap &&
 968             (!pt->sync_switch ||
 969              ptq->switch_state != INTEL_PT_SS_EXPECTING_SWITCH_EVENT)) {
 970                 const struct intel_pt_state *state;
 971                 int ret;
 972
 973                 if (pt->timeless_decoding)
 974                         return 0;
 975
 976                 intel_pt_log("queue %u getting timestamp\n", queue_nr);
 977                 intel_pt_log("queue %u decoding cpu %d pid %d tid %d\n",
 978                              queue_nr, ptq->cpu, ptq->pid, ptq->tid);
 979                 while (1) {
 980                         state = intel_pt_decode(ptq->decoder);
 981                         if (state->err) {
 982                                 if (state->err == INTEL_PT_ERR_NODATA) {
 983                                         intel_pt_log("queue %u has no timestamp\n",
 984                                                      queue_nr);
 985                                         return 0;
 986                                 }
 987                                 continue;
 988                         }
 989                         if (state->timestamp)
 990                                 break;
 991                 }
 992
 993                 ptq->timestamp = state->timestamp;
 994                 intel_pt_log("queue %u timestamp 0x%" PRIx64 "\n",
 995                              queue_nr, ptq->timestamp);
 996                 ptq->state = state;
 997                 ptq->have_sample = true;
 998                 intel_pt_sample_flags(ptq);
 999                 ret = auxtrace_heap__add(&pt->heap, queue_nr, ptq->timestamp);
1000                 if (ret)
1001                         return ret;
1002                 ptq->on_heap = true;
1003         }
1004
1005         return 0;
1006 }
1007
1008 static int intel_pt_setup_queues(struct intel_pt *pt)
1009 {
1010         unsigned int i;
1011         int ret;
1012
1013         for (i = 0; i < pt->queues.nr_queues; i++) {
1014                 ret = intel_pt_setup_queue(pt, &pt->queues.queue_array[i], i);
1015                 if (ret)
1016                         return ret;
1017         }
1018         return 0;
1019 }
1020
1021 static inline void intel_pt_copy_last_branch_rb(struct intel_pt_queue *ptq)
1022 {
1023         struct branch_stack *bs_src = ptq->last_branch_rb;
1024         struct branch_stack *bs_dst = ptq->last_branch;
1025         size_t nr = 0;
1026
1027         bs_dst->nr = bs_src->nr;
1028
1029         if (!bs_src->nr)
1030                 return;
1031
1032         nr = ptq->pt->synth_opts.last_branch_sz - ptq->last_branch_pos;
1033         memcpy(&bs_dst->entries[0],
1034                &bs_src->entries[ptq->last_branch_pos],
1035                sizeof(struct branch_entry) * nr);
1036
1037         if (bs_src->nr >= ptq->pt->synth_opts.last_branch_sz) {
1038                 memcpy(&bs_dst->entries[nr],
1039                        &bs_src->entries[0],
1040                        sizeof(struct branch_entry) * ptq->last_branch_pos);
1041         }
1042 }
1043
1044 static inline void intel_pt_reset_last_branch_rb(struct intel_pt_queue *ptq)
1045 {
1046         ptq->last_branch_pos = 0;
1047         ptq->last_branch_rb->nr = 0;
1048 }
1049
1050 static void intel_pt_update_last_branch_rb(struct intel_pt_queue *ptq)
1051 {
1052         const struct intel_pt_state *state = ptq->state;
1053         struct branch_stack *bs = ptq->last_branch_rb;
1054         struct branch_entry *be;
1055
1056         if (!ptq->last_branch_pos)
1057                 ptq->last_branch_pos = ptq->pt->synth_opts.last_branch_sz;
1058
1059         ptq->last_branch_pos -= 1;
1060
1061         be              = &bs->entries[ptq->last_branch_pos];
1062         be->from        = state->from_ip;
1063         be->to          = state->to_ip;
1064         be->flags.abort = !!(state->flags & INTEL_PT_ABORT_TX);
1065         be->flags.in_tx = !!(state->flags & INTEL_PT_IN_TX);
1066         /* No support for mispredict */
1067         be->flags.mispred = ptq->pt->mispred_all;
1068
1069         if (bs->nr < ptq->pt->synth_opts.last_branch_sz)
1070                 bs->nr += 1;
1071 }
1072
1073 static inline bool intel_pt_skip_event(struct intel_pt *pt)
1074 {
1075         return pt->synth_opts.initial_skip &&
1076                pt->num_events++ < pt->synth_opts.initial_skip;
1077 }
1078
1079 static void intel_pt_prep_b_sample(struct intel_pt *pt,
1080                                    struct intel_pt_queue *ptq,
1081                                    union perf_event *event,
1082                                    struct perf_sample *sample)
1083 {
1084         event->sample.header.type = PERF_RECORD_SAMPLE;
1085         event->sample.header.misc = PERF_RECORD_MISC_USER;
1086         event->sample.header.size = sizeof(struct perf_event_header);
1087
1088         if (!pt->timeless_decoding)
1089                 sample->time = tsc_to_perf_time(ptq->timestamp, &pt->tc);
1090
1091         sample->cpumode = PERF_RECORD_MISC_USER;
1092         sample->ip = ptq->state->from_ip;
1093         sample->pid = ptq->pid;
1094         sample->tid = ptq->tid;
1095         sample->addr = ptq->state->to_ip;
1096         sample->period = 1;
1097         sample->cpu = ptq->cpu;
1098         sample->flags = ptq->flags;
1099         sample->insn_len = ptq->insn_len;
1100         memcpy(sample->insn, ptq->insn, INTEL_PT_INSN_BUF_SZ);
1101 }
1102
1103 static int intel_pt_inject_event(union perf_event *event,
1104                                  struct perf_sample *sample, u64 type,
1105                                  bool swapped)
1106 {
1107         event->header.size = perf_event__sample_event_size(sample, type, 0);
1108         return perf_event__synthesize_sample(event, type, 0, sample, swapped);
1109 }
1110
1111 static inline int intel_pt_opt_inject(struct intel_pt *pt,
1112                                       union perf_event *event,
1113                                       struct perf_sample *sample, u64 type)
1114 {
1115         if (!pt->synth_opts.inject)
1116                 return 0;
1117
1118         return intel_pt_inject_event(event, sample, type, pt->synth_needs_swap);
1119 }
1120
1121 static int intel_pt_deliver_synth_b_event(struct intel_pt *pt,
1122                                           union perf_event *event,
1123                                           struct perf_sample *sample, u64 type)
1124 {
1125         int ret;
1126
1127         ret = intel_pt_opt_inject(pt, event, sample, type);
1128         if (ret)
1129                 return ret;
1130
1131         ret = perf_session__deliver_synth_event(pt->session, event, sample);
1132         if (ret)
1133                 pr_err("Intel PT: failed to deliver event, error %d\n", ret);
1134
1135         return ret;
1136 }
1137
1138 static int intel_pt_synth_branch_sample(struct intel_pt_queue *ptq)
1139 {
1140         struct intel_pt *pt = ptq->pt;
1141         union perf_event *event = ptq->event_buf;
1142         struct perf_sample sample = { .ip = 0, };
1143         struct dummy_branch_stack {
1144                 u64                     nr;
1145                 struct branch_entry     entries;
1146         } dummy_bs;
1147
1148         if (pt->branches_filter && !(pt->branches_filter & ptq->flags))
1149                 return 0;
1150
1151         if (intel_pt_skip_event(pt))
1152                 return 0;
1153
1154         intel_pt_prep_b_sample(pt, ptq, event, &sample);
1155
1156         sample.id = ptq->pt->branches_id;
1157         sample.stream_id = ptq->pt->branches_id;
1158
1159         /*
1160          * perf report cannot handle events without a branch stack when using
1161          * SORT_MODE__BRANCH so make a dummy one.
1162          */
1163         if (pt->synth_opts.last_branch && sort__mode == SORT_MODE__BRANCH) {
1164                 dummy_bs = (struct dummy_branch_stack){
1165                         .nr = 1,
1166                         .entries = {
1167                                 .from = sample.ip,
1168                                 .to = sample.addr,
1169                         },
1170                 };
1171                 sample.branch_stack = (struct branch_stack *)&dummy_bs;
1172         }
1173
1174         return intel_pt_deliver_synth_b_event(pt, event, &sample,
1175                                               pt->branches_sample_type);
1176 }
1177
1178 static void intel_pt_prep_sample(struct intel_pt *pt,
1179                                  struct intel_pt_queue *ptq,
1180                                  union perf_event *event,
1181                                  struct perf_sample *sample)
1182 {
1183         intel_pt_prep_b_sample(pt, ptq, event, sample);
1184
1185         if (pt->synth_opts.callchain) {
1186                 thread_stack__sample(ptq->thread, ptq->chain,
1187                                      pt->synth_opts.callchain_sz, sample->ip);
1188                 sample->callchain = ptq->chain;
1189         }
1190
1191         if (pt->synth_opts.last_branch) {
1192                 intel_pt_copy_last_branch_rb(ptq);
1193                 sample->branch_stack = ptq->last_branch;
1194         }
1195 }
1196
1197 static inline int intel_pt_deliver_synth_event(struct intel_pt *pt,
1198                                                struct intel_pt_queue *ptq,
1199                                                union perf_event *event,
1200                                                struct perf_sample *sample,
1201                                                u64 type)
1202 {
1203         int ret;
1204
1205         ret = intel_pt_deliver_synth_b_event(pt, event, sample, type);
1206
1207         if (pt->synth_opts.last_branch)
1208                 intel_pt_reset_last_branch_rb(ptq);
1209
1210         return ret;
1211 }
1212
1213 static int intel_pt_synth_instruction_sample(struct intel_pt_queue *ptq)
1214 {
1215         struct intel_pt *pt = ptq->pt;
1216         union perf_event *event = ptq->event_buf;
1217         struct perf_sample sample = { .ip = 0, };
1218
1219         if (intel_pt_skip_event(pt))
1220                 return 0;
1221
1222         intel_pt_prep_sample(pt, ptq, event, &sample);
1223
1224         sample.id = ptq->pt->instructions_id;
1225         sample.stream_id = ptq->pt->instructions_id;
1226         sample.period = ptq->state->tot_insn_cnt - ptq->last_insn_cnt;
1227
1228         ptq->last_insn_cnt = ptq->state->tot_insn_cnt;
1229
1230         return intel_pt_deliver_synth_event(pt, ptq, event, &sample,
1231                                             pt->instructions_sample_type);
1232 }
1233
1234 static int intel_pt_synth_transaction_sample(struct intel_pt_queue *ptq)
1235 {
1236         struct intel_pt *pt = ptq->pt;
1237         union perf_event *event = ptq->event_buf;
1238         struct perf_sample sample = { .ip = 0, };
1239
1240         if (intel_pt_skip_event(pt))
1241                 return 0;
1242
1243         intel_pt_prep_sample(pt, ptq, event, &sample);
1244
1245         sample.id = ptq->pt->transactions_id;
1246         sample.stream_id = ptq->pt->transactions_id;
1247
1248         return intel_pt_deliver_synth_event(pt, ptq, event, &sample,
1249                                             pt->transactions_sample_type);
1250 }
1251
1252 static void intel_pt_prep_p_sample(struct intel_pt *pt,
1253                                    struct intel_pt_queue *ptq,
1254                                    union perf_event *event,
1255                                    struct perf_sample *sample)
1256 {
1257         intel_pt_prep_sample(pt, ptq, event, sample);
1258
1259         /*
1260          * Zero IP is used to mean "trace start" but that is not the case for
1261          * power or PTWRITE events with no IP, so clear the flags.
1262          */
1263         if (!sample->ip)
1264                 sample->flags = 0;
1265 }
1266
1267 static int intel_pt_synth_ptwrite_sample(struct intel_pt_queue *ptq)
1268 {
1269         struct intel_pt *pt = ptq->pt;
1270         union perf_event *event = ptq->event_buf;
1271         struct perf_sample sample = { .ip = 0, };
1272         struct perf_synth_intel_ptwrite raw;
1273
1274         if (intel_pt_skip_event(pt))
1275                 return 0;
1276
1277         intel_pt_prep_p_sample(pt, ptq, event, &sample);
1278
1279         sample.id = ptq->pt->ptwrites_id;
1280         sample.stream_id = ptq->pt->ptwrites_id;
1281
1282         raw.flags = 0;
1283         raw.ip = !!(ptq->state->flags & INTEL_PT_FUP_IP);
1284         raw.payload = cpu_to_le64(ptq->state->ptw_payload);
1285
1286         sample.raw_size = perf_synth__raw_size(raw);
1287         sample.raw_data = perf_synth__raw_data(&raw);
1288
1289         return intel_pt_deliver_synth_event(pt, ptq, event, &sample,
1290                                             pt->ptwrites_sample_type);
1291 }
1292
1293 static int intel_pt_synth_cbr_sample(struct intel_pt_queue *ptq)
1294 {
1295         struct intel_pt *pt = ptq->pt;
1296         union perf_event *event = ptq->event_buf;
1297         struct perf_sample sample = { .ip = 0, };
1298         struct perf_synth_intel_cbr raw;
1299         u32 flags;
1300
1301         if (intel_pt_skip_event(pt))
1302                 return 0;
1303
1304         intel_pt_prep_p_sample(pt, ptq, event, &sample);
1305
1306         sample.id = ptq->pt->cbr_id;
1307         sample.stream_id = ptq->pt->cbr_id;
1308
1309         flags = (u16)ptq->state->cbr_payload | (pt->max_non_turbo_ratio << 16);
1310         raw.flags = cpu_to_le32(flags);
1311         raw.freq = cpu_to_le32(raw.cbr * pt->cbr2khz);
1312         raw.reserved3 = 0;
1313
1314         sample.raw_size = perf_synth__raw_size(raw);
1315         sample.raw_data = perf_synth__raw_data(&raw);
1316
1317         return intel_pt_deliver_synth_event(pt, ptq, event, &sample,
1318                                             pt->pwr_events_sample_type);
1319 }
1320
1321 static int intel_pt_synth_mwait_sample(struct intel_pt_queue *ptq)
1322 {
1323         struct intel_pt *pt = ptq->pt;
1324         union perf_event *event = ptq->event_buf;
1325         struct perf_sample sample = { .ip = 0, };
1326         struct perf_synth_intel_mwait raw;
1327
1328         if (intel_pt_skip_event(pt))
1329                 return 0;
1330
1331         intel_pt_prep_p_sample(pt, ptq, event, &sample);
1332
1333         sample.id = ptq->pt->mwait_id;
1334         sample.stream_id = ptq->pt->mwait_id;
1335
1336         raw.reserved = 0;
1337         raw.payload = cpu_to_le64(ptq->state->mwait_payload);
1338
1339         sample.raw_size = perf_synth__raw_size(raw);
1340         sample.raw_data = perf_synth__raw_data(&raw);
1341
1342         return intel_pt_deliver_synth_event(pt, ptq, event, &sample,
1343                                             pt->pwr_events_sample_type);
1344 }
1345
1346 static int intel_pt_synth_pwre_sample(struct intel_pt_queue *ptq)
1347 {
1348         struct intel_pt *pt = ptq->pt;
1349         union perf_event *event = ptq->event_buf;
1350         struct perf_sample sample = { .ip = 0, };
1351         struct perf_synth_intel_pwre raw;
1352
1353         if (intel_pt_skip_event(pt))
1354                 return 0;
1355
1356         intel_pt_prep_p_sample(pt, ptq, event, &sample);
1357
1358         sample.id = ptq->pt->pwre_id;
1359         sample.stream_id = ptq->pt->pwre_id;
1360
1361         raw.reserved = 0;
1362         raw.payload = cpu_to_le64(ptq->state->pwre_payload);
1363
1364         sample.raw_size = perf_synth__raw_size(raw);
1365         sample.raw_data = perf_synth__raw_data(&raw);
1366
1367         return intel_pt_deliver_synth_event(pt, ptq, event, &sample,
1368                                             pt->pwr_events_sample_type);
1369 }
1370
1371 static int intel_pt_synth_exstop_sample(struct intel_pt_queue *ptq)
1372 {
1373         struct intel_pt *pt = ptq->pt;
1374         union perf_event *event = ptq->event_buf;
1375         struct perf_sample sample = { .ip = 0, };
1376         struct perf_synth_intel_exstop raw;
1377
1378         if (intel_pt_skip_event(pt))
1379                 return 0;
1380
1381         intel_pt_prep_p_sample(pt, ptq, event, &sample);
1382
1383         sample.id = ptq->pt->exstop_id;
1384         sample.stream_id = ptq->pt->exstop_id;
1385
1386         raw.flags = 0;
1387         raw.ip = !!(ptq->state->flags & INTEL_PT_FUP_IP);
1388
1389         sample.raw_size = perf_synth__raw_size(raw);
1390         sample.raw_data = perf_synth__raw_data(&raw);
1391
1392         return intel_pt_deliver_synth_event(pt, ptq, event, &sample,
1393                                             pt->pwr_events_sample_type);
1394 }
1395
1396 static int intel_pt_synth_pwrx_sample(struct intel_pt_queue *ptq)
1397 {
1398         struct intel_pt *pt = ptq->pt;
1399         union perf_event *event = ptq->event_buf;
1400         struct perf_sample sample = { .ip = 0, };
1401         struct perf_synth_intel_pwrx raw;
1402
1403         if (intel_pt_skip_event(pt))
1404                 return 0;
1405
1406         intel_pt_prep_p_sample(pt, ptq, event, &sample);
1407
1408         sample.id = ptq->pt->pwrx_id;
1409         sample.stream_id = ptq->pt->pwrx_id;
1410
1411         raw.reserved = 0;
1412         raw.payload = cpu_to_le64(ptq->state->pwrx_payload);
1413
1414         sample.raw_size = perf_synth__raw_size(raw);
1415         sample.raw_data = perf_synth__raw_data(&raw);
1416
1417         return intel_pt_deliver_synth_event(pt, ptq, event, &sample,
1418                                             pt->pwr_events_sample_type);
1419 }
1420
1421 static int intel_pt_synth_error(struct intel_pt *pt, int code, int cpu,
1422                                 pid_t pid, pid_t tid, u64 ip)
1423 {
1424         union perf_event event;
1425         char msg[MAX_AUXTRACE_ERROR_MSG];
1426         int err;
1427
1428         intel_pt__strerror(code, msg, MAX_AUXTRACE_ERROR_MSG);
1429
1430         auxtrace_synth_error(&event.auxtrace_error, PERF_AUXTRACE_ERROR_ITRACE,
1431                              code, cpu, pid, tid, ip, msg);
1432
1433         err = perf_session__deliver_synth_event(pt->session, &event, NULL);
1434         if (err)
1435                 pr_err("Intel Processor Trace: failed to deliver error event, error %d\n",
1436                        err);
1437
1438         return err;
1439 }
1440
1441 static int intel_pt_next_tid(struct intel_pt *pt, struct intel_pt_queue *ptq)
1442 {
1443         struct auxtrace_queue *queue;
1444         pid_t tid = ptq->next_tid;
1445         int err;
1446
1447         if (tid == -1)
1448                 return 0;
1449
1450         intel_pt_log("switch: cpu %d tid %d\n", ptq->cpu, tid);
1451
1452         err = machine__set_current_tid(pt->machine, ptq->cpu, -1, tid);
1453
1454         queue = &pt->queues.queue_array[ptq->queue_nr];
1455         intel_pt_set_pid_tid_cpu(pt, queue);
1456
1457         ptq->next_tid = -1;
1458
1459         return err;
1460 }
1461
1462 static inline bool intel_pt_is_switch_ip(struct intel_pt_queue *ptq, u64 ip)
1463 {
1464         struct intel_pt *pt = ptq->pt;
1465
1466         return ip == pt->switch_ip &&
1467                (ptq->flags & PERF_IP_FLAG_BRANCH) &&
1468                !(ptq->flags & (PERF_IP_FLAG_CONDITIONAL | PERF_IP_FLAG_ASYNC |
1469                                PERF_IP_FLAG_INTERRUPT | PERF_IP_FLAG_TX_ABORT));
1470 }
1471
1472 #define INTEL_PT_PWR_EVT (INTEL_PT_MWAIT_OP | INTEL_PT_PWR_ENTRY | \
1473                           INTEL_PT_EX_STOP | INTEL_PT_PWR_EXIT | \
1474                           INTEL_PT_CBR_CHG)
1475
1476 static int intel_pt_sample(struct intel_pt_queue *ptq)
1477 {
1478         const struct intel_pt_state *state = ptq->state;
1479         struct intel_pt *pt = ptq->pt;
1480         int err;
1481
1482         if (!ptq->have_sample)
1483                 return 0;
1484
1485         ptq->have_sample = false;
1486
1487         if (pt->sample_pwr_events && (state->type & INTEL_PT_PWR_EVT)) {
1488                 if (state->type & INTEL_PT_CBR_CHG) {
1489                         err = intel_pt_synth_cbr_sample(ptq);
1490                         if (err)
1491                                 return err;
1492                 }
1493                 if (state->type & INTEL_PT_MWAIT_OP) {
1494                         err = intel_pt_synth_mwait_sample(ptq);
1495                         if (err)
1496                                 return err;
1497                 }
1498                 if (state->type & INTEL_PT_PWR_ENTRY) {
1499                         err = intel_pt_synth_pwre_sample(ptq);
1500                         if (err)
1501                                 return err;
1502                 }
1503                 if (state->type & INTEL_PT_EX_STOP) {
1504                         err = intel_pt_synth_exstop_sample(ptq);
1505                         if (err)
1506                                 return err;
1507                 }
1508                 if (state->type & INTEL_PT_PWR_EXIT) {
1509                         err = intel_pt_synth_pwrx_sample(ptq);
1510                         if (err)
1511                                 return err;
1512                 }
1513         }
1514
1515         if (pt->sample_instructions && (state->type & INTEL_PT_INSTRUCTION)) {
1516                 err = intel_pt_synth_instruction_sample(ptq);
1517                 if (err)
1518                         return err;
1519         }
1520
1521         if (pt->sample_transactions && (state->type & INTEL_PT_TRANSACTION)) {
1522                 err = intel_pt_synth_transaction_sample(ptq);
1523                 if (err)
1524                         return err;
1525         }
1526
1527         if (pt->sample_ptwrites && (state->type & INTEL_PT_PTW)) {
1528                 err = intel_pt_synth_ptwrite_sample(ptq);
1529                 if (err)
1530                         return err;
1531         }
1532
1533         if (!(state->type & INTEL_PT_BRANCH))
1534                 return 0;
1535
1536         if (pt->synth_opts.callchain || pt->synth_opts.thread_stack)
1537                 thread_stack__event(ptq->thread, ptq->flags, state->from_ip,
1538                                     state->to_ip, ptq->insn_len,
1539                                     state->trace_nr);
1540         else
1541                 thread_stack__set_trace_nr(ptq->thread, state->trace_nr);
1542
1543         if (pt->sample_branches) {
1544                 err = intel_pt_synth_branch_sample(ptq);
1545                 if (err)
1546                         return err;
1547         }
1548
1549         if (pt->synth_opts.last_branch)
1550                 intel_pt_update_last_branch_rb(ptq);
1551
1552         if (!pt->sync_switch)
1553                 return 0;
1554
1555         if (intel_pt_is_switch_ip(ptq, state->to_ip)) {
1556                 switch (ptq->switch_state) {
1557                 case INTEL_PT_SS_UNKNOWN:
1558                 case INTEL_PT_SS_EXPECTING_SWITCH_IP:
1559                         err = intel_pt_next_tid(pt, ptq);
1560                         if (err)
1561                                 return err;
1562                         ptq->switch_state = INTEL_PT_SS_TRACING;
1563                         break;
1564                 default:
1565                         ptq->switch_state = INTEL_PT_SS_EXPECTING_SWITCH_EVENT;
1566                         return 1;
1567                 }
1568         } else if (!state->to_ip) {
1569                 ptq->switch_state = INTEL_PT_SS_NOT_TRACING;
1570         } else if (ptq->switch_state == INTEL_PT_SS_NOT_TRACING) {
1571                 ptq->switch_state = INTEL_PT_SS_UNKNOWN;
1572         } else if (ptq->switch_state == INTEL_PT_SS_UNKNOWN &&
1573                    state->to_ip == pt->ptss_ip &&
1574                    (ptq->flags & PERF_IP_FLAG_CALL)) {
1575                 ptq->switch_state = INTEL_PT_SS_TRACING;
1576         }
1577
1578         return 0;
1579 }
1580
1581 static u64 intel_pt_switch_ip(struct intel_pt *pt, u64 *ptss_ip)
1582 {
1583         struct machine *machine = pt->machine;
1584         struct map *map;
1585         struct symbol *sym, *start;
1586         u64 ip, switch_ip = 0;
1587         const char *ptss;
1588
1589         if (ptss_ip)
1590                 *ptss_ip = 0;
1591
1592         map = machine__kernel_map(machine);
1593         if (!map)
1594                 return 0;
1595
1596         if (map__load(map))
1597                 return 0;
1598
1599         start = dso__first_symbol(map->dso, MAP__FUNCTION);
1600
1601         for (sym = start; sym; sym = dso__next_symbol(sym)) {
1602                 if (sym->binding == STB_GLOBAL &&
1603                     !strcmp(sym->name, "__switch_to")) {
1604                         ip = map->unmap_ip(map, sym->start);
1605                         if (ip >= map->start && ip < map->end) {
1606                                 switch_ip = ip;
1607                                 break;
1608                         }
1609                 }
1610         }
1611
1612         if (!switch_ip || !ptss_ip)
1613                 return 0;
1614
1615         if (pt->have_sched_switch == 1)
1616                 ptss = "perf_trace_sched_switch";
1617         else
1618                 ptss = "__perf_event_task_sched_out";
1619
1620         for (sym = start; sym; sym = dso__next_symbol(sym)) {
1621                 if (!strcmp(sym->name, ptss)) {
1622                         ip = map->unmap_ip(map, sym->start);
1623                         if (ip >= map->start && ip < map->end) {
1624                                 *ptss_ip = ip;
1625                                 break;
1626                         }
1627                 }
1628         }
1629
1630         return switch_ip;
1631 }
1632
1633 static int intel_pt_run_decoder(struct intel_pt_queue *ptq, u64 *timestamp)
1634 {
1635         const struct intel_pt_state *state = ptq->state;
1636         struct intel_pt *pt = ptq->pt;
1637         int err;
1638
1639         if (!pt->kernel_start) {
1640                 pt->kernel_start = machine__kernel_start(pt->machine);
1641                 if (pt->per_cpu_mmaps &&
1642                     (pt->have_sched_switch == 1 || pt->have_sched_switch == 3) &&
1643                     !pt->timeless_decoding && intel_pt_tracing_kernel(pt) &&
1644                     !pt->sampling_mode) {
1645                         pt->switch_ip = intel_pt_switch_ip(pt, &pt->ptss_ip);
1646                         if (pt->switch_ip) {
1647                                 intel_pt_log("switch_ip: %"PRIx64" ptss_ip: %"PRIx64"\n",
1648                                              pt->switch_ip, pt->ptss_ip);
1649                                 pt->sync_switch = true;
1650                         }
1651                 }
1652         }
1653
1654         intel_pt_log("queue %u decoding cpu %d pid %d tid %d\n",
1655                      ptq->queue_nr, ptq->cpu, ptq->pid, ptq->tid);
1656         while (1) {
1657                 err = intel_pt_sample(ptq);
1658                 if (err)
1659                         return err;
1660
1661                 state = intel_pt_decode(ptq->decoder);
1662                 if (state->err) {
1663                         if (state->err == INTEL_PT_ERR_NODATA)
1664                                 return 1;
1665                         if (pt->sync_switch &&
1666                             state->from_ip >= pt->kernel_start) {
1667                                 pt->sync_switch = false;
1668                                 intel_pt_next_tid(pt, ptq);
1669                         }
1670                         if (pt->synth_opts.errors) {
1671                                 err = intel_pt_synth_error(pt, state->err,
1672                                                            ptq->cpu, ptq->pid,
1673                                                            ptq->tid,
1674                                                            state->from_ip);
1675                                 if (err)
1676                                         return err;
1677                         }
1678                         continue;
1679                 }
1680
1681                 ptq->state = state;
1682                 ptq->have_sample = true;
1683                 intel_pt_sample_flags(ptq);
1684
1685                 /* Use estimated TSC upon return to user space */
1686                 if (pt->est_tsc &&
1687                     (state->from_ip >= pt->kernel_start || !state->from_ip) &&
1688                     state->to_ip && state->to_ip < pt->kernel_start) {
1689                         intel_pt_log("TSC %"PRIx64" est. TSC %"PRIx64"\n",
1690                                      state->timestamp, state->est_timestamp);
1691                         ptq->timestamp = state->est_timestamp;
1692                 /* Use estimated TSC in unknown switch state */
1693                 } else if (pt->sync_switch &&
1694                            ptq->switch_state == INTEL_PT_SS_UNKNOWN &&
1695                            intel_pt_is_switch_ip(ptq, state->to_ip) &&
1696                            ptq->next_tid == -1) {
1697                         intel_pt_log("TSC %"PRIx64" est. TSC %"PRIx64"\n",
1698                                      state->timestamp, state->est_timestamp);
1699                         ptq->timestamp = state->est_timestamp;
1700                 } else if (state->timestamp > ptq->timestamp) {
1701                         ptq->timestamp = state->timestamp;
1702                 }
1703
1704                 if (!pt->timeless_decoding && ptq->timestamp >= *timestamp) {
1705                         *timestamp = ptq->timestamp;
1706                         return 0;
1707                 }
1708         }
1709         return 0;
1710 }
1711
1712 static inline int intel_pt_update_queues(struct intel_pt *pt)
1713 {
1714         if (pt->queues.new_data) {
1715                 pt->queues.new_data = false;
1716                 return intel_pt_setup_queues(pt);
1717         }
1718         return 0;
1719 }
1720
1721 static int intel_pt_process_queues(struct intel_pt *pt, u64 timestamp)
1722 {
1723         unsigned int queue_nr;
1724         u64 ts;
1725         int ret;
1726
1727         while (1) {
1728                 struct auxtrace_queue *queue;
1729                 struct intel_pt_queue *ptq;
1730
1731                 if (!pt->heap.heap_cnt)
1732                         return 0;
1733
1734                 if (pt->heap.heap_array[0].ordinal >= timestamp)
1735                         return 0;
1736
1737                 queue_nr = pt->heap.heap_array[0].queue_nr;
1738                 queue = &pt->queues.queue_array[queue_nr];
1739                 ptq = queue->priv;
1740
1741                 intel_pt_log("queue %u processing 0x%" PRIx64 " to 0x%" PRIx64 "\n",
1742                              queue_nr, pt->heap.heap_array[0].ordinal,
1743                              timestamp);
1744
1745                 auxtrace_heap__pop(&pt->heap);
1746
1747                 if (pt->heap.heap_cnt) {
1748                         ts = pt->heap.heap_array[0].ordinal + 1;
1749                         if (ts > timestamp)
1750                                 ts = timestamp;
1751                 } else {
1752                         ts = timestamp;
1753                 }
1754
1755                 intel_pt_set_pid_tid_cpu(pt, queue);
1756
1757                 ret = intel_pt_run_decoder(ptq, &ts);
1758
1759                 if (ret < 0) {
1760                         auxtrace_heap__add(&pt->heap, queue_nr, ts);
1761                         return ret;
1762                 }
1763
1764                 if (!ret) {
1765                         ret = auxtrace_heap__add(&pt->heap, queue_nr, ts);
1766                         if (ret < 0)
1767                                 return ret;
1768                 } else {
1769                         ptq->on_heap = false;
1770                 }
1771         }
1772
1773         return 0;
1774 }
1775
1776 static int intel_pt_process_timeless_queues(struct intel_pt *pt, pid_t tid,
1777                                             u64 time_)
1778 {
1779         struct auxtrace_queues *queues = &pt->queues;
1780         unsigned int i;
1781         u64 ts = 0;
1782
1783         for (i = 0; i < queues->nr_queues; i++) {
1784                 struct auxtrace_queue *queue = &pt->queues.queue_array[i];
1785                 struct intel_pt_queue *ptq = queue->priv;
1786
1787                 if (ptq && (tid == -1 || ptq->tid == tid)) {
1788                         ptq->time = time_;
1789                         intel_pt_set_pid_tid_cpu(pt, queue);
1790                         intel_pt_run_decoder(ptq, &ts);
1791                 }
1792         }
1793         return 0;
1794 }
1795
1796 static int intel_pt_lost(struct intel_pt *pt, struct perf_sample *sample)
1797 {
1798         return intel_pt_synth_error(pt, INTEL_PT_ERR_LOST, sample->cpu,
1799                                     sample->pid, sample->tid, 0);
1800 }
1801
1802 static struct intel_pt_queue *intel_pt_cpu_to_ptq(struct intel_pt *pt, int cpu)
1803 {
1804         unsigned i, j;
1805
1806         if (cpu < 0 || !pt->queues.nr_queues)
1807                 return NULL;
1808
1809         if ((unsigned)cpu >= pt->queues.nr_queues)
1810                 i = pt->queues.nr_queues - 1;
1811         else
1812                 i = cpu;
1813
1814         if (pt->queues.queue_array[i].cpu == cpu)
1815                 return pt->queues.queue_array[i].priv;
1816
1817         for (j = 0; i > 0; j++) {
1818                 if (pt->queues.queue_array[--i].cpu == cpu)
1819                         return pt->queues.queue_array[i].priv;
1820         }
1821
1822         for (; j < pt->queues.nr_queues; j++) {
1823                 if (pt->queues.queue_array[j].cpu == cpu)
1824                         return pt->queues.queue_array[j].priv;
1825         }
1826
1827         return NULL;
1828 }
1829
1830 static int intel_pt_sync_switch(struct intel_pt *pt, int cpu, pid_t tid,
1831                                 u64 timestamp)
1832 {
1833         struct intel_pt_queue *ptq;
1834         int err;
1835
1836         if (!pt->sync_switch)
1837                 return 1;
1838
1839         ptq = intel_pt_cpu_to_ptq(pt, cpu);
1840         if (!ptq)
1841                 return 1;
1842
1843         switch (ptq->switch_state) {
1844         case INTEL_PT_SS_NOT_TRACING:
1845                 ptq->next_tid = -1;
1846                 break;
1847         case INTEL_PT_SS_UNKNOWN:
1848         case INTEL_PT_SS_TRACING:
1849                 ptq->next_tid = tid;
1850                 ptq->switch_state = INTEL_PT_SS_EXPECTING_SWITCH_IP;
1851                 return 0;
1852         case INTEL_PT_SS_EXPECTING_SWITCH_EVENT:
1853                 if (!ptq->on_heap) {
1854                         ptq->timestamp = perf_time_to_tsc(timestamp,
1855                                                           &pt->tc);
1856                         err = auxtrace_heap__add(&pt->heap, ptq->queue_nr,
1857                                                  ptq->timestamp);
1858                         if (err)
1859                                 return err;
1860                         ptq->on_heap = true;
1861                 }
1862                 ptq->switch_state = INTEL_PT_SS_TRACING;
1863                 break;
1864         case INTEL_PT_SS_EXPECTING_SWITCH_IP:
1865                 ptq->next_tid = tid;
1866                 intel_pt_log("ERROR: cpu %d expecting switch ip\n", cpu);
1867                 break;
1868         default:
1869                 break;
1870         }
1871
1872         return 1;
1873 }
1874
1875 static int intel_pt_process_switch(struct intel_pt *pt,
1876                                    struct perf_sample *sample)
1877 {
1878         struct perf_evsel *evsel;
1879         pid_t tid;
1880         int cpu, ret;
1881
1882         evsel = perf_evlist__id2evsel(pt->session->evlist, sample->id);
1883         if (evsel != pt->switch_evsel)
1884                 return 0;
1885
1886         tid = perf_evsel__intval(evsel, sample, "next_pid");
1887         cpu = sample->cpu;
1888
1889         intel_pt_log("sched_switch: cpu %d tid %d time %"PRIu64" tsc %#"PRIx64"\n",
1890                      cpu, tid, sample->time, perf_time_to_tsc(sample->time,
1891                      &pt->tc));
1892
1893         ret = intel_pt_sync_switch(pt, cpu, tid, sample->time);
1894         if (ret <= 0)
1895                 return ret;
1896
1897         return machine__set_current_tid(pt->machine, cpu, -1, tid);
1898 }
1899
1900 static int intel_pt_context_switch(struct intel_pt *pt, union perf_event *event,
1901                                    struct perf_sample *sample)
1902 {
1903         bool out = event->header.misc & PERF_RECORD_MISC_SWITCH_OUT;
1904         pid_t pid, tid;
1905         int cpu, ret;
1906
1907         cpu = sample->cpu;
1908
1909         if (pt->have_sched_switch == 3) {
1910                 if (!out)
1911                         return 0;
1912                 if (event->header.type != PERF_RECORD_SWITCH_CPU_WIDE) {
1913                         pr_err("Expecting CPU-wide context switch event\n");
1914                         return -EINVAL;
1915                 }
1916                 pid = event->context_switch.next_prev_pid;
1917                 tid = event->context_switch.next_prev_tid;
1918         } else {
1919                 if (out)
1920                         return 0;
1921                 pid = sample->pid;
1922                 tid = sample->tid;
1923         }
1924
1925         if (tid == -1) {
1926                 pr_err("context_switch event has no tid\n");
1927                 return -EINVAL;
1928         }
1929
1930         intel_pt_log("context_switch: cpu %d pid %d tid %d time %"PRIu64" tsc %#"PRIx64"\n",
1931                      cpu, pid, tid, sample->time, perf_time_to_tsc(sample->time,
1932                      &pt->tc));
1933
1934         ret = intel_pt_sync_switch(pt, cpu, tid, sample->time);
1935         if (ret <= 0)
1936                 return ret;
1937
1938         return machine__set_current_tid(pt->machine, cpu, pid, tid);
1939 }
1940
1941 static int intel_pt_process_itrace_start(struct intel_pt *pt,
1942                                          union perf_event *event,
1943                                          struct perf_sample *sample)
1944 {
1945         if (!pt->per_cpu_mmaps)
1946                 return 0;
1947
1948         intel_pt_log("itrace_start: cpu %d pid %d tid %d time %"PRIu64" tsc %#"PRIx64"\n",
1949                      sample->cpu, event->itrace_start.pid,
1950                      event->itrace_start.tid, sample->time,
1951                      perf_time_to_tsc(sample->time, &pt->tc));
1952
1953         return machine__set_current_tid(pt->machine, sample->cpu,
1954                                         event->itrace_start.pid,
1955                                         event->itrace_start.tid);
1956 }
1957
1958 static int intel_pt_process_event(struct perf_session *session,
1959                                   union perf_event *event,
1960                                   struct perf_sample *sample,
1961                                   struct perf_tool *tool)
1962 {
1963         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
1964                                            auxtrace);
1965         u64 timestamp;
1966         int err = 0;
1967
1968         if (dump_trace)
1969                 return 0;
1970
1971         if (!tool->ordered_events) {
1972                 pr_err("Intel Processor Trace requires ordered events\n");
1973                 return -EINVAL;
1974         }
1975
1976         if (sample->time && sample->time != (u64)-1)
1977                 timestamp = perf_time_to_tsc(sample->time, &pt->tc);
1978         else
1979                 timestamp = 0;
1980
1981         if (timestamp || pt->timeless_decoding) {
1982                 err = intel_pt_update_queues(pt);
1983                 if (err)
1984                         return err;
1985         }
1986
1987         if (pt->timeless_decoding) {
1988                 if (event->header.type == PERF_RECORD_EXIT) {
1989                         err = intel_pt_process_timeless_queues(pt,
1990                                                                event->fork.tid,
1991                                                                sample->time);
1992                 }
1993         } else if (timestamp) {
1994                 err = intel_pt_process_queues(pt, timestamp);
1995         }
1996         if (err)
1997                 return err;
1998
1999         if (event->header.type == PERF_RECORD_AUX &&
2000             (event->aux.flags & PERF_AUX_FLAG_TRUNCATED) &&
2001             pt->synth_opts.errors) {
2002                 err = intel_pt_lost(pt, sample);
2003                 if (err)
2004                         return err;
2005         }
2006
2007         if (pt->switch_evsel && event->header.type == PERF_RECORD_SAMPLE)
2008                 err = intel_pt_process_switch(pt, sample);
2009         else if (event->header.type == PERF_RECORD_ITRACE_START)
2010                 err = intel_pt_process_itrace_start(pt, event, sample);
2011         else if (event->header.type == PERF_RECORD_SWITCH ||
2012                  event->header.type == PERF_RECORD_SWITCH_CPU_WIDE)
2013                 err = intel_pt_context_switch(pt, event, sample);
2014
2015         intel_pt_log("event %s (%u): cpu %d time %"PRIu64" tsc %#"PRIx64"\n",
2016                      perf_event__name(event->header.type), event->header.type,
2017                      sample->cpu, sample->time, timestamp);
2018
2019         return err;
2020 }
2021
2022 static int intel_pt_flush(struct perf_session *session, struct perf_tool *tool)
2023 {
2024         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
2025                                            auxtrace);
2026         int ret;
2027
2028         if (dump_trace)
2029                 return 0;
2030
2031         if (!tool->ordered_events)
2032                 return -EINVAL;
2033
2034         ret = intel_pt_update_queues(pt);
2035         if (ret < 0)
2036                 return ret;
2037
2038         if (pt->timeless_decoding)
2039                 return intel_pt_process_timeless_queues(pt, -1,
2040                                                         MAX_TIMESTAMP - 1);
2041
2042         return intel_pt_process_queues(pt, MAX_TIMESTAMP);
2043 }
2044
2045 static void intel_pt_free_events(struct perf_session *session)
2046 {
2047         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
2048                                            auxtrace);
2049         struct auxtrace_queues *queues = &pt->queues;
2050         unsigned int i;
2051
2052         for (i = 0; i < queues->nr_queues; i++) {
2053                 intel_pt_free_queue(queues->queue_array[i].priv);
2054                 queues->queue_array[i].priv = NULL;
2055         }
2056         intel_pt_log_disable();
2057         auxtrace_queues__free(queues);
2058 }
2059
2060 static void intel_pt_free(struct perf_session *session)
2061 {
2062         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
2063                                            auxtrace);
2064
2065         auxtrace_heap__free(&pt->heap);
2066         intel_pt_free_events(session);
2067         session->auxtrace = NULL;
2068         thread__put(pt->unknown_thread);
2069         addr_filters__exit(&pt->filts);
2070         zfree(&pt->filter);
2071         free(pt);
2072 }
2073
2074 static int intel_pt_process_auxtrace_event(struct perf_session *session,
2075                                            union perf_event *event,
2076                                            struct perf_tool *tool __maybe_unused)
2077 {
2078         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
2079                                            auxtrace);
2080
2081         if (pt->sampling_mode)
2082                 return 0;
2083
2084         if (!pt->data_queued) {
2085                 struct auxtrace_buffer *buffer;
2086                 off_t data_offset;
2087                 int fd = perf_data_file__fd(session->file);
2088                 int err;
2089
2090                 if (perf_data_file__is_pipe(session->file)) {
2091                         data_offset = 0;
2092                 } else {
2093                         data_offset = lseek(fd, 0, SEEK_CUR);
2094                         if (data_offset == -1)
2095                                 return -errno;
2096                 }
2097
2098                 err = auxtrace_queues__add_event(&pt->queues, session, event,
2099                                                  data_offset, &buffer);
2100                 if (err)
2101                         return err;
2102
2103                 /* Dump here now we have copied a piped trace out of the pipe */
2104                 if (dump_trace) {
2105                         if (auxtrace_buffer__get_data(buffer, fd)) {
2106                                 intel_pt_dump_event(pt, buffer->data,
2107                                                     buffer->size);
2108                                 auxtrace_buffer__put_data(buffer);
2109                         }
2110                 }
2111         }
2112
2113         return 0;
2114 }
2115
2116 struct intel_pt_synth {
2117         struct perf_tool dummy_tool;
2118         struct perf_session *session;
2119 };
2120
2121 static int intel_pt_event_synth(struct perf_tool *tool,
2122                                 union perf_event *event,
2123                                 struct perf_sample *sample __maybe_unused,
2124                                 struct machine *machine __maybe_unused)
2125 {
2126         struct intel_pt_synth *intel_pt_synth =
2127                         container_of(tool, struct intel_pt_synth, dummy_tool);
2128
2129         return perf_session__deliver_synth_event(intel_pt_synth->session, event,
2130                                                  NULL);
2131 }
2132
2133 static int intel_pt_synth_event(struct perf_session *session, const char *name,
2134                                 struct perf_event_attr *attr, u64 id)
2135 {
2136         struct intel_pt_synth intel_pt_synth;
2137         int err;
2138
2139         pr_debug("Synthesizing '%s' event with id %" PRIu64 " sample type %#" PRIx64 "\n",
2140                  name, id, (u64)attr->sample_type);
2141
2142         memset(&intel_pt_synth, 0, sizeof(struct intel_pt_synth));
2143         intel_pt_synth.session = session;
2144
2145         err = perf_event__synthesize_attr(&intel_pt_synth.dummy_tool, attr, 1,
2146                                           &id, intel_pt_event_synth);
2147         if (err)
2148                 pr_err("%s: failed to synthesize '%s' event type\n",
2149                        __func__, name);
2150
2151         return err;
2152 }
2153
2154 static void intel_pt_set_event_name(struct perf_evlist *evlist, u64 id,
2155                                     const char *name)
2156 {
2157         struct perf_evsel *evsel;
2158
2159         evlist__for_each_entry(evlist, evsel) {
2160                 if (evsel->id && evsel->id[0] == id) {
2161                         if (evsel->name)
2162                                 zfree(&evsel->name);
2163                         evsel->name = strdup(name);
2164                         break;
2165                 }
2166         }
2167 }
2168
2169 static struct perf_evsel *intel_pt_evsel(struct intel_pt *pt,
2170                                          struct perf_evlist *evlist)
2171 {
2172         struct perf_evsel *evsel;
2173
2174         evlist__for_each_entry(evlist, evsel) {
2175                 if (evsel->attr.type == pt->pmu_type && evsel->ids)
2176                         return evsel;
2177         }
2178
2179         return NULL;
2180 }
2181
2182 static int intel_pt_synth_events(struct intel_pt *pt,
2183                                  struct perf_session *session)
2184 {
2185         struct perf_evlist *evlist = session->evlist;
2186         struct perf_evsel *evsel = intel_pt_evsel(pt, evlist);
2187         struct perf_event_attr attr;
2188         u64 id;
2189         int err;
2190
2191         if (!evsel) {
2192                 pr_debug("There are no selected events with Intel Processor Trace data\n");
2193                 return 0;
2194         }
2195
2196         memset(&attr, 0, sizeof(struct perf_event_attr));
2197         attr.size = sizeof(struct perf_event_attr);
2198         attr.type = PERF_TYPE_HARDWARE;
2199         attr.sample_type = evsel->attr.sample_type & PERF_SAMPLE_MASK;
2200         attr.sample_type |= PERF_SAMPLE_IP | PERF_SAMPLE_TID |
2201                             PERF_SAMPLE_PERIOD;
2202         if (pt->timeless_decoding)
2203                 attr.sample_type &= ~(u64)PERF_SAMPLE_TIME;
2204         else
2205                 attr.sample_type |= PERF_SAMPLE_TIME;
2206         if (!pt->per_cpu_mmaps)
2207                 attr.sample_type &= ~(u64)PERF_SAMPLE_CPU;
2208         attr.exclude_user = evsel->attr.exclude_user;
2209         attr.exclude_kernel = evsel->attr.exclude_kernel;
2210         attr.exclude_hv = evsel->attr.exclude_hv;
2211         attr.exclude_host = evsel->attr.exclude_host;
2212         attr.exclude_guest = evsel->attr.exclude_guest;
2213         attr.sample_id_all = evsel->attr.sample_id_all;
2214         attr.read_format = evsel->attr.read_format;
2215
2216         id = evsel->id[0] + 1000000000;
2217         if (!id)
2218                 id = 1;
2219
2220         if (pt->synth_opts.branches) {
2221                 attr.config = PERF_COUNT_HW_BRANCH_INSTRUCTIONS;
2222                 attr.sample_period = 1;
2223                 attr.sample_type |= PERF_SAMPLE_ADDR;
2224                 err = intel_pt_synth_event(session, "branches", &attr, id);
2225                 if (err)
2226                         return err;
2227                 pt->sample_branches = true;
2228                 pt->branches_sample_type = attr.sample_type;
2229                 pt->branches_id = id;
2230                 id += 1;
2231                 attr.sample_type &= ~(u64)PERF_SAMPLE_ADDR;
2232         }
2233
2234         if (pt->synth_opts.callchain)
2235                 attr.sample_type |= PERF_SAMPLE_CALLCHAIN;
2236         if (pt->synth_opts.last_branch)
2237                 attr.sample_type |= PERF_SAMPLE_BRANCH_STACK;
2238
2239         if (pt->synth_opts.instructions) {
2240                 attr.config = PERF_COUNT_HW_INSTRUCTIONS;
2241                 if (pt->synth_opts.period_type == PERF_ITRACE_PERIOD_NANOSECS)
2242                         attr.sample_period =
2243                                 intel_pt_ns_to_ticks(pt, pt->synth_opts.period);
2244                 else
2245                         attr.sample_period = pt->synth_opts.period;
2246                 err = intel_pt_synth_event(session, "instructions", &attr, id);
2247                 if (err)
2248                         return err;
2249                 pt->sample_instructions = true;
2250                 pt->instructions_sample_type = attr.sample_type;
2251                 pt->instructions_id = id;
2252                 id += 1;
2253         }
2254
2255         attr.sample_type &= ~(u64)PERF_SAMPLE_PERIOD;
2256         attr.sample_period = 1;
2257
2258         if (pt->synth_opts.transactions) {
2259                 attr.config = PERF_COUNT_HW_INSTRUCTIONS;
2260                 err = intel_pt_synth_event(session, "transactions", &attr, id);
2261                 if (err)
2262                         return err;
2263                 pt->sample_transactions = true;
2264                 pt->transactions_sample_type = attr.sample_type;
2265                 pt->transactions_id = id;
2266                 intel_pt_set_event_name(evlist, id, "transactions");
2267                 id += 1;
2268         }
2269
2270         attr.type = PERF_TYPE_SYNTH;
2271         attr.sample_type |= PERF_SAMPLE_RAW;
2272
2273         if (pt->synth_opts.ptwrites) {
2274                 attr.config = PERF_SYNTH_INTEL_PTWRITE;
2275                 err = intel_pt_synth_event(session, "ptwrite", &attr, id);
2276                 if (err)
2277                         return err;
2278                 pt->sample_ptwrites = true;
2279                 pt->ptwrites_sample_type = attr.sample_type;
2280                 pt->ptwrites_id = id;
2281                 intel_pt_set_event_name(evlist, id, "ptwrite");
2282                 id += 1;
2283         }
2284
2285         if (pt->synth_opts.pwr_events) {
2286                 pt->sample_pwr_events = true;
2287                 pt->pwr_events_sample_type = attr.sample_type;
2288
2289                 attr.config = PERF_SYNTH_INTEL_CBR;
2290                 err = intel_pt_synth_event(session, "cbr", &attr, id);
2291                 if (err)
2292                         return err;
2293                 pt->cbr_id = id;
2294                 intel_pt_set_event_name(evlist, id, "cbr");
2295                 id += 1;
2296         }
2297
2298         if (pt->synth_opts.pwr_events && (evsel->attr.config & 0x10)) {
2299                 attr.config = PERF_SYNTH_INTEL_MWAIT;
2300                 err = intel_pt_synth_event(session, "mwait", &attr, id);
2301                 if (err)
2302                         return err;
2303                 pt->mwait_id = id;
2304                 intel_pt_set_event_name(evlist, id, "mwait");
2305                 id += 1;
2306
2307                 attr.config = PERF_SYNTH_INTEL_PWRE;
2308                 err = intel_pt_synth_event(session, "pwre", &attr, id);
2309                 if (err)
2310                         return err;
2311                 pt->pwre_id = id;
2312                 intel_pt_set_event_name(evlist, id, "pwre");
2313                 id += 1;
2314
2315                 attr.config = PERF_SYNTH_INTEL_EXSTOP;
2316                 err = intel_pt_synth_event(session, "exstop", &attr, id);
2317                 if (err)
2318                         return err;
2319                 pt->exstop_id = id;
2320                 intel_pt_set_event_name(evlist, id, "exstop");
2321                 id += 1;
2322
2323                 attr.config = PERF_SYNTH_INTEL_PWRX;
2324                 err = intel_pt_synth_event(session, "pwrx", &attr, id);
2325                 if (err)
2326                         return err;
2327                 pt->pwrx_id = id;
2328                 intel_pt_set_event_name(evlist, id, "pwrx");
2329                 id += 1;
2330         }
2331
2332         pt->synth_needs_swap = evsel->needs_swap;
2333
2334         return 0;
2335 }
2336
2337 static struct perf_evsel *intel_pt_find_sched_switch(struct perf_evlist *evlist)
2338 {
2339         struct perf_evsel *evsel;
2340
2341         evlist__for_each_entry_reverse(evlist, evsel) {
2342                 const char *name = perf_evsel__name(evsel);
2343
2344                 if (!strcmp(name, "sched:sched_switch"))
2345                         return evsel;
2346         }
2347
2348         return NULL;
2349 }
2350
2351 static bool intel_pt_find_switch(struct perf_evlist *evlist)
2352 {
2353         struct perf_evsel *evsel;
2354
2355         evlist__for_each_entry(evlist, evsel) {
2356                 if (evsel->attr.context_switch)
2357                         return true;
2358         }
2359
2360         return false;
2361 }
2362
2363 static int intel_pt_perf_config(const char *var, const char *value, void *data)
2364 {
2365         struct intel_pt *pt = data;
2366
2367         if (!strcmp(var, "intel-pt.mispred-all"))
2368                 pt->mispred_all = perf_config_bool(var, value);
2369
2370         return 0;
2371 }
2372
2373 static const char * const intel_pt_info_fmts[] = {
2374         [INTEL_PT_PMU_TYPE]             = "  PMU Type            %"PRId64"\n",
2375         [INTEL_PT_TIME_SHIFT]           = "  Time Shift          %"PRIu64"\n",
2376         [INTEL_PT_TIME_MULT]            = "  Time Muliplier      %"PRIu64"\n",
2377         [INTEL_PT_TIME_ZERO]            = "  Time Zero           %"PRIu64"\n",
2378         [INTEL_PT_CAP_USER_TIME_ZERO]   = "  Cap Time Zero       %"PRId64"\n",
2379         [INTEL_PT_TSC_BIT]              = "  TSC bit             %#"PRIx64"\n",
2380         [INTEL_PT_NORETCOMP_BIT]        = "  NoRETComp bit       %#"PRIx64"\n",
2381         [INTEL_PT_HAVE_SCHED_SWITCH]    = "  Have sched_switch   %"PRId64"\n",
2382         [INTEL_PT_SNAPSHOT_MODE]        = "  Snapshot mode       %"PRId64"\n",
2383         [INTEL_PT_PER_CPU_MMAPS]        = "  Per-cpu maps        %"PRId64"\n",
2384         [INTEL_PT_MTC_BIT]              = "  MTC bit             %#"PRIx64"\n",
2385         [INTEL_PT_TSC_CTC_N]            = "  TSC:CTC numerator   %"PRIu64"\n",
2386         [INTEL_PT_TSC_CTC_D]            = "  TSC:CTC denominator %"PRIu64"\n",
2387         [INTEL_PT_CYC_BIT]              = "  CYC bit             %#"PRIx64"\n",
2388         [INTEL_PT_MAX_NONTURBO_RATIO]   = "  Max non-turbo ratio %"PRIu64"\n",
2389         [INTEL_PT_FILTER_STR_LEN]       = "  Filter string len.  %"PRIu64"\n",
2390 };
2391
2392 static void intel_pt_print_info(u64 *arr, int start, int finish)
2393 {
2394         int i;
2395
2396         if (!dump_trace)
2397                 return;
2398
2399         for (i = start; i <= finish; i++)
2400                 fprintf(stdout, intel_pt_info_fmts[i], arr[i]);
2401 }
2402
2403 static void intel_pt_print_info_str(const char *name, const char *str)
2404 {
2405         if (!dump_trace)
2406                 return;
2407
2408         fprintf(stdout, "  %-20s%s\n", name, str ? str : "");
2409 }
2410
2411 static bool intel_pt_has(struct auxtrace_info_event *auxtrace_info, int pos)
2412 {
2413         return auxtrace_info->header.size >=
2414                 sizeof(struct auxtrace_info_event) + (sizeof(u64) * (pos + 1));
2415 }
2416
2417 int intel_pt_process_auxtrace_info(union perf_event *event,
2418                                    struct perf_session *session)
2419 {
2420         struct auxtrace_info_event *auxtrace_info = &event->auxtrace_info;
2421         size_t min_sz = sizeof(u64) * INTEL_PT_PER_CPU_MMAPS;
2422         struct intel_pt *pt;
2423         void *info_end;
2424         u64 *info;
2425         int err;
2426
2427         if (auxtrace_info->header.size < sizeof(struct auxtrace_info_event) +
2428                                         min_sz)
2429                 return -EINVAL;
2430
2431         pt = zalloc(sizeof(struct intel_pt));
2432         if (!pt)
2433                 return -ENOMEM;
2434
2435         addr_filters__init(&pt->filts);
2436
2437         err = perf_config(intel_pt_perf_config, pt);
2438         if (err)
2439                 goto err_free;
2440
2441         err = auxtrace_queues__init(&pt->queues);
2442         if (err)
2443                 goto err_free;
2444
2445         intel_pt_log_set_name(INTEL_PT_PMU_NAME);
2446
2447         pt->session = session;
2448         pt->machine = &session->machines.host; /* No kvm support */
2449         pt->auxtrace_type = auxtrace_info->type;
2450         pt->pmu_type = auxtrace_info->priv[INTEL_PT_PMU_TYPE];
2451         pt->tc.time_shift = auxtrace_info->priv[INTEL_PT_TIME_SHIFT];
2452         pt->tc.time_mult = auxtrace_info->priv[INTEL_PT_TIME_MULT];
2453         pt->tc.time_zero = auxtrace_info->priv[INTEL_PT_TIME_ZERO];
2454         pt->cap_user_time_zero = auxtrace_info->priv[INTEL_PT_CAP_USER_TIME_ZERO];
2455         pt->tsc_bit = auxtrace_info->priv[INTEL_PT_TSC_BIT];
2456         pt->noretcomp_bit = auxtrace_info->priv[INTEL_PT_NORETCOMP_BIT];
2457         pt->have_sched_switch = auxtrace_info->priv[INTEL_PT_HAVE_SCHED_SWITCH];
2458         pt->snapshot_mode = auxtrace_info->priv[INTEL_PT_SNAPSHOT_MODE];
2459         pt->per_cpu_mmaps = auxtrace_info->priv[INTEL_PT_PER_CPU_MMAPS];
2460         intel_pt_print_info(&auxtrace_info->priv[0], INTEL_PT_PMU_TYPE,
2461                             INTEL_PT_PER_CPU_MMAPS);
2462
2463         if (intel_pt_has(auxtrace_info, INTEL_PT_CYC_BIT)) {
2464                 pt->mtc_bit = auxtrace_info->priv[INTEL_PT_MTC_BIT];
2465                 pt->mtc_freq_bits = auxtrace_info->priv[INTEL_PT_MTC_FREQ_BITS];
2466                 pt->tsc_ctc_ratio_n = auxtrace_info->priv[INTEL_PT_TSC_CTC_N];
2467                 pt->tsc_ctc_ratio_d = auxtrace_info->priv[INTEL_PT_TSC_CTC_D];
2468                 pt->cyc_bit = auxtrace_info->priv[INTEL_PT_CYC_BIT];
2469                 intel_pt_print_info(&auxtrace_info->priv[0], INTEL_PT_MTC_BIT,
2470                                     INTEL_PT_CYC_BIT);
2471         }
2472
2473         if (intel_pt_has(auxtrace_info, INTEL_PT_MAX_NONTURBO_RATIO)) {
2474                 pt->max_non_turbo_ratio =
2475                         auxtrace_info->priv[INTEL_PT_MAX_NONTURBO_RATIO];
2476                 intel_pt_print_info(&auxtrace_info->priv[0],
2477                                     INTEL_PT_MAX_NONTURBO_RATIO,
2478                                     INTEL_PT_MAX_NONTURBO_RATIO);
2479         }
2480
2481         info = &auxtrace_info->priv[INTEL_PT_FILTER_STR_LEN] + 1;
2482         info_end = (void *)info + auxtrace_info->header.size;
2483
2484         if (intel_pt_has(auxtrace_info, INTEL_PT_FILTER_STR_LEN)) {
2485                 size_t len;
2486
2487                 len = auxtrace_info->priv[INTEL_PT_FILTER_STR_LEN];
2488                 intel_pt_print_info(&auxtrace_info->priv[0],
2489                                     INTEL_PT_FILTER_STR_LEN,
2490                                     INTEL_PT_FILTER_STR_LEN);
2491                 if (len) {
2492                         const char *filter = (const char *)info;
2493
2494                         len = roundup(len + 1, 8);
2495                         info += len >> 3;
2496                         if ((void *)info > info_end) {
2497                                 pr_err("%s: bad filter string length\n", __func__);
2498                                 err = -EINVAL;
2499                                 goto err_free_queues;
2500                         }
2501                         pt->filter = memdup(filter, len);
2502                         if (!pt->filter) {
2503                                 err = -ENOMEM;
2504                                 goto err_free_queues;
2505                         }
2506                         if (session->header.needs_swap)
2507                                 mem_bswap_64(pt->filter, len);
2508                         if (pt->filter[len - 1]) {
2509                                 pr_err("%s: filter string not null terminated\n", __func__);
2510                                 err = -EINVAL;
2511                                 goto err_free_queues;
2512                         }
2513                         err = addr_filters__parse_bare_filter(&pt->filts,
2514                                                               filter);
2515                         if (err)
2516                                 goto err_free_queues;
2517                 }
2518                 intel_pt_print_info_str("Filter string", pt->filter);
2519         }
2520
2521         pt->timeless_decoding = intel_pt_timeless_decoding(pt);
2522         pt->have_tsc = intel_pt_have_tsc(pt);
2523         pt->sampling_mode = false;
2524         pt->est_tsc = !pt->timeless_decoding;
2525
2526         pt->unknown_thread = thread__new(999999999, 999999999);
2527         if (!pt->unknown_thread) {
2528                 err = -ENOMEM;
2529                 goto err_free_queues;
2530         }
2531
2532         /*
2533          * Since this thread will not be kept in any rbtree not in a
2534          * list, initialize its list node so that at thread__put() the
2535          * current thread lifetime assuption is kept and we don't segfault
2536          * at list_del_init().
2537          */
2538         INIT_LIST_HEAD(&pt->unknown_thread->node);
2539
2540         err = thread__set_comm(pt->unknown_thread, "unknown", 0);
2541         if (err)
2542                 goto err_delete_thread;
2543         if (thread__init_map_groups(pt->unknown_thread, pt->machine)) {
2544                 err = -ENOMEM;
2545                 goto err_delete_thread;
2546         }
2547
2548         pt->auxtrace.process_event = intel_pt_process_event;
2549         pt->auxtrace.process_auxtrace_event = intel_pt_process_auxtrace_event;
2550         pt->auxtrace.flush_events = intel_pt_flush;
2551         pt->auxtrace.free_events = intel_pt_free_events;
2552         pt->auxtrace.free = intel_pt_free;
2553         session->auxtrace = &pt->auxtrace;
2554
2555         if (dump_trace)
2556                 return 0;
2557
2558         if (pt->have_sched_switch == 1) {
2559                 pt->switch_evsel = intel_pt_find_sched_switch(session->evlist);
2560                 if (!pt->switch_evsel) {
2561                         pr_err("%s: missing sched_switch event\n", __func__);
2562                         err = -EINVAL;
2563                         goto err_delete_thread;
2564                 }
2565         } else if (pt->have_sched_switch == 2 &&
2566                    !intel_pt_find_switch(session->evlist)) {
2567                 pr_err("%s: missing context_switch attribute flag\n", __func__);
2568                 err = -EINVAL;
2569                 goto err_delete_thread;
2570         }
2571
2572         if (session->itrace_synth_opts && session->itrace_synth_opts->set) {
2573                 pt->synth_opts = *session->itrace_synth_opts;
2574         } else {
2575                 itrace_synth_opts__set_default(&pt->synth_opts);
2576                 if (use_browser != -1) {
2577                         pt->synth_opts.branches = false;
2578                         pt->synth_opts.callchain = true;
2579                 }
2580                 if (session->itrace_synth_opts)
2581                         pt->synth_opts.thread_stack =
2582                                 session->itrace_synth_opts->thread_stack;
2583         }
2584
2585         if (pt->synth_opts.log)
2586                 intel_pt_log_enable();
2587
2588         /* Maximum non-turbo ratio is TSC freq / 100 MHz */
2589         if (pt->tc.time_mult) {
2590                 u64 tsc_freq = intel_pt_ns_to_ticks(pt, 1000000000);
2591
2592                 if (!pt->max_non_turbo_ratio)
2593                         pt->max_non_turbo_ratio =
2594                                         (tsc_freq + 50000000) / 100000000;
2595                 intel_pt_log("TSC frequency %"PRIu64"\n", tsc_freq);
2596                 intel_pt_log("Maximum non-turbo ratio %u\n",
2597                              pt->max_non_turbo_ratio);
2598                 pt->cbr2khz = tsc_freq / pt->max_non_turbo_ratio / 1000;
2599         }
2600
2601         if (pt->synth_opts.calls)
2602                 pt->branches_filter |= PERF_IP_FLAG_CALL | PERF_IP_FLAG_ASYNC |
2603                                        PERF_IP_FLAG_TRACE_END;
2604         if (pt->synth_opts.returns)
2605                 pt->branches_filter |= PERF_IP_FLAG_RETURN |
2606                                        PERF_IP_FLAG_TRACE_BEGIN;
2607
2608         if (pt->synth_opts.callchain && !symbol_conf.use_callchain) {
2609                 symbol_conf.use_callchain = true;
2610                 if (callchain_register_param(&callchain_param) < 0) {
2611                         symbol_conf.use_callchain = false;
2612                         pt->synth_opts.callchain = false;
2613                 }
2614         }
2615
2616         err = intel_pt_synth_events(pt, session);
2617         if (err)
2618                 goto err_delete_thread;
2619
2620         err = auxtrace_queues__process_index(&pt->queues, session);
2621         if (err)
2622                 goto err_delete_thread;
2623
2624         if (pt->queues.populated)
2625                 pt->data_queued = true;
2626
2627         if (pt->timeless_decoding)
2628                 pr_debug2("Intel PT decoding without timestamps\n");
2629
2630         return 0;
2631
2632 err_delete_thread:
2633         thread__zput(pt->unknown_thread);
2634 err_free_queues:
2635         intel_pt_log_disable();
2636         auxtrace_queues__free(&pt->queues);
2637         session->auxtrace = NULL;
2638 err_free:
2639         addr_filters__exit(&pt->filts);
2640         zfree(&pt->filter);
2641         free(pt);
2642         return err;
2643 }