tools/perf/util/intel-pt.c

   1 /*
   2  * intel_pt.c: Intel Processor Trace support
   3  * Copyright (c) 2013-2015, Intel Corporation.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  *
  14  */
  15
  16 #include <stdio.h>
  17 #include <stdbool.h>
  18 #include <errno.h>
  19 #include <linux/kernel.h>
  20 #include <linux/types.h>
  21
  22 #include "../perf.h"
  23 #include "session.h"
  24 #include "machine.h"
  25 #include "sort.h"
  26 #include "tool.h"
  27 #include "event.h"
  28 #include "evlist.h"
  29 #include "evsel.h"
  30 #include "map.h"
  31 #include "color.h"
  32 #include "util.h"
  33 #include "thread.h"
  34 #include "thread-stack.h"
  35 #include "symbol.h"
  36 #include "callchain.h"
  37 #include "dso.h"
  38 #include "debug.h"
  39 #include "auxtrace.h"
  40 #include "tsc.h"
  41 #include "intel-pt.h"
  42
  43 #include "intel-pt-decoder/intel-pt-log.h"
  44 #include "intel-pt-decoder/intel-pt-decoder.h"
  45 #include "intel-pt-decoder/intel-pt-insn-decoder.h"
  46 #include "intel-pt-decoder/intel-pt-pkt-decoder.h"
  47
  48 #define MAX_TIMESTAMP (~0ULL)
  49
  50 struct intel_pt {
  51         struct auxtrace auxtrace;
  52         struct auxtrace_queues queues;
  53         struct auxtrace_heap heap;
  54         u32 auxtrace_type;
  55         struct perf_session *session;
  56         struct machine *machine;
  57         struct perf_evsel *switch_evsel;
  58         struct thread *unknown_thread;
  59         bool timeless_decoding;
  60         bool sampling_mode;
  61         bool snapshot_mode;
  62         bool per_cpu_mmaps;
  63         bool have_tsc;
  64         bool data_queued;
  65         bool est_tsc;
  66         bool sync_switch;
  67         bool mispred_all;
  68         int have_sched_switch;
  69         u32 pmu_type;
  70         u64 kernel_start;
  71         u64 switch_ip;
  72         u64 ptss_ip;
  73
  74         struct perf_tsc_conversion tc;
  75         bool cap_user_time_zero;
  76
  77         struct itrace_synth_opts synth_opts;
  78
  79         bool sample_instructions;
  80         u64 instructions_sample_type;
  81         u64 instructions_sample_period;
  82         u64 instructions_id;
  83
  84         bool sample_branches;
  85         u32 branches_filter;
  86         u64 branches_sample_type;
  87         u64 branches_id;
  88
  89         bool sample_transactions;
  90         u64 transactions_sample_type;
  91         u64 transactions_id;
  92
  93         bool synth_needs_swap;
  94
  95         u64 tsc_bit;
  96         u64 mtc_bit;
  97         u64 mtc_freq_bits;
  98         u32 tsc_ctc_ratio_n;
  99         u32 tsc_ctc_ratio_d;
 100         u64 cyc_bit;
 101         u64 noretcomp_bit;
 102         unsigned max_non_turbo_ratio;
 103 };
 104
 105 enum switch_state {
 106         INTEL_PT_SS_NOT_TRACING,
 107         INTEL_PT_SS_UNKNOWN,
 108         INTEL_PT_SS_TRACING,
 109         INTEL_PT_SS_EXPECTING_SWITCH_EVENT,
 110         INTEL_PT_SS_EXPECTING_SWITCH_IP,
 111 };
 112
 113 struct intel_pt_queue {
 114         struct intel_pt *pt;
 115         unsigned int queue_nr;
 116         struct auxtrace_buffer *buffer;
 117         void *decoder;
 118         const struct intel_pt_state *state;
 119         struct ip_callchain *chain;
 120         struct branch_stack *last_branch;
 121         struct branch_stack *last_branch_rb;
 122         size_t last_branch_pos;
 123         union perf_event *event_buf;
 124         bool on_heap;
 125         bool stop;
 126         bool step_through_buffers;
 127         bool use_buffer_pid_tid;
 128         pid_t pid, tid;
 129         int cpu;
 130         int switch_state;
 131         pid_t next_tid;
 132         struct thread *thread;
 133         bool exclude_kernel;
 134         bool have_sample;
 135         u64 time;
 136         u64 timestamp;
 137         u32 flags;
 138         u16 insn_len;
 139         u64 last_insn_cnt;
 140 };
 141
 142 static void intel_pt_dump(struct intel_pt *pt __maybe_unused,
 143                           unsigned char *buf, size_t len)
 144 {
 145         struct intel_pt_pkt packet;
 146         size_t pos = 0;
 147         int ret, pkt_len, i;
 148         char desc[INTEL_PT_PKT_DESC_MAX];
 149         const char *color = PERF_COLOR_BLUE;
 150
 151         color_fprintf(stdout, color,
 152                       ". ... Intel Processor Trace data: size %zu bytes\n",
 153                       len);
 154
 155         while (len) {
 156                 ret = intel_pt_get_packet(buf, len, &packet);
 157                 if (ret > 0)
 158                         pkt_len = ret;
 159                 else
 160                         pkt_len = 1;
 161                 printf(".");
 162                 color_fprintf(stdout, color, "  %08x: ", pos);
 163                 for (i = 0; i < pkt_len; i++)
 164                         color_fprintf(stdout, color, " %02x", buf[i]);
 165                 for (; i < 16; i++)
 166                         color_fprintf(stdout, color, "   ");
 167                 if (ret > 0) {
 168                         ret = intel_pt_pkt_desc(&packet, desc,
 169                                                 INTEL_PT_PKT_DESC_MAX);
 170                         if (ret > 0)
 171                                 color_fprintf(stdout, color, " %s\n", desc);
 172                 } else {
 173                         color_fprintf(stdout, color, " Bad packet!\n");
 174                 }
 175                 pos += pkt_len;
 176                 buf += pkt_len;
 177                 len -= pkt_len;
 178         }
 179 }
 180
 181 static void intel_pt_dump_event(struct intel_pt *pt, unsigned char *buf,
 182                                 size_t len)
 183 {
 184         printf(".\n");
 185         intel_pt_dump(pt, buf, len);
 186 }
 187
 188 static int intel_pt_do_fix_overlap(struct intel_pt *pt, struct auxtrace_buffer *a,
 189                                    struct auxtrace_buffer *b)
 190 {
 191         void *start;
 192
 193         start = intel_pt_find_overlap(a->data, a->size, b->data, b->size,
 194                                       pt->have_tsc);
 195         if (!start)
 196                 return -EINVAL;
 197         b->use_size = b->data + b->size - start;
 198         b->use_data = start;
 199         return 0;
 200 }
 201
 202 static void intel_pt_use_buffer_pid_tid(struct intel_pt_queue *ptq,
 203                                         struct auxtrace_queue *queue,
 204                                         struct auxtrace_buffer *buffer)
 205 {
 206         if (queue->cpu == -1 && buffer->cpu != -1)
 207                 ptq->cpu = buffer->cpu;
 208
 209         ptq->pid = buffer->pid;
 210         ptq->tid = buffer->tid;
 211
 212         intel_pt_log("queue %u cpu %d pid %d tid %d\n",
 213                      ptq->queue_nr, ptq->cpu, ptq->pid, ptq->tid);
 214
 215         thread__zput(ptq->thread);
 216
 217         if (ptq->tid != -1) {
 218                 if (ptq->pid != -1)
 219                         ptq->thread = machine__findnew_thread(ptq->pt->machine,
 220                                                               ptq->pid,
 221                                                               ptq->tid);
 222                 else
 223                         ptq->thread = machine__find_thread(ptq->pt->machine, -1,
 224                                                            ptq->tid);
 225         }
 226 }
 227
 228 /* This function assumes data is processed sequentially only */
 229 static int intel_pt_get_trace(struct intel_pt_buffer *b, void *data)
 230 {
 231         struct intel_pt_queue *ptq = data;
 232         struct auxtrace_buffer *buffer = ptq->buffer, *old_buffer = buffer;
 233         struct auxtrace_queue *queue;
 234
 235         if (ptq->stop) {
 236                 b->len = 0;
 237                 return 0;
 238         }
 239
 240         queue = &ptq->pt->queues.queue_array[ptq->queue_nr];
 241
 242         buffer = auxtrace_buffer__next(queue, buffer);
 243         if (!buffer) {
 244                 if (old_buffer)
 245                         auxtrace_buffer__drop_data(old_buffer);
 246                 b->len = 0;
 247                 return 0;
 248         }
 249
 250         ptq->buffer = buffer;
 251
 252         if (!buffer->data) {
 253                 int fd = perf_data_file__fd(ptq->pt->session->file);
 254
 255                 buffer->data = auxtrace_buffer__get_data(buffer, fd);
 256                 if (!buffer->data)
 257                         return -ENOMEM;
 258         }
 259
 260         if (ptq->pt->snapshot_mode && !buffer->consecutive && old_buffer &&
 261             intel_pt_do_fix_overlap(ptq->pt, old_buffer, buffer))
 262                 return -ENOMEM;
 263
 264         if (old_buffer)
 265                 auxtrace_buffer__drop_data(old_buffer);
 266
 267         if (buffer->use_data) {
 268                 b->len = buffer->use_size;
 269                 b->buf = buffer->use_data;
 270         } else {
 271                 b->len = buffer->size;
 272                 b->buf = buffer->data;
 273         }
 274         b->ref_timestamp = buffer->reference;
 275
 276         if (!old_buffer || ptq->pt->sampling_mode || (ptq->pt->snapshot_mode &&
 277                                                       !buffer->consecutive)) {
 278                 b->consecutive = false;
 279                 b->trace_nr = buffer->buffer_nr + 1;
 280         } else {
 281                 b->consecutive = true;
 282         }
 283
 284         if (ptq->use_buffer_pid_tid && (ptq->pid != buffer->pid ||
 285                                         ptq->tid != buffer->tid))
 286                 intel_pt_use_buffer_pid_tid(ptq, queue, buffer);
 287
 288         if (ptq->step_through_buffers)
 289                 ptq->stop = true;
 290
 291         if (!b->len)
 292                 return intel_pt_get_trace(b, data);
 293
 294         return 0;
 295 }
 296
 297 struct intel_pt_cache_entry {
 298         struct auxtrace_cache_entry     entry;
 299         u64                             insn_cnt;
 300         u64                             byte_cnt;
 301         enum intel_pt_insn_op           op;
 302         enum intel_pt_insn_branch       branch;
 303         int                             length;
 304         int32_t                         rel;
 305 };
 306
 307 static int intel_pt_config_div(const char *var, const char *value, void *data)
 308 {
 309         int *d = data;
 310         long val;
 311
 312         if (!strcmp(var, "intel-pt.cache-divisor")) {
 313                 val = strtol(value, NULL, 0);
 314                 if (val > 0 && val <= INT_MAX)
 315                         *d = val;
 316         }
 317
 318         return 0;
 319 }
 320
 321 static int intel_pt_cache_divisor(void)
 322 {
 323         static int d;
 324
 325         if (d)
 326                 return d;
 327
 328         perf_config(intel_pt_config_div, &d);
 329
 330         if (!d)
 331                 d = 64;
 332
 333         return d;
 334 }
 335
 336 static unsigned int intel_pt_cache_size(struct dso *dso,
 337                                         struct machine *machine)
 338 {
 339         off_t size;
 340
 341         size = dso__data_size(dso, machine);
 342         size /= intel_pt_cache_divisor();
 343         if (size < 1000)
 344                 return 10;
 345         if (size > (1 << 21))
 346                 return 21;
 347         return 32 - __builtin_clz(size);
 348 }
 349
 350 static struct auxtrace_cache *intel_pt_cache(struct dso *dso,
 351                                              struct machine *machine)
 352 {
 353         struct auxtrace_cache *c;
 354         unsigned int bits;
 355
 356         if (dso->auxtrace_cache)
 357                 return dso->auxtrace_cache;
 358
 359         bits = intel_pt_cache_size(dso, machine);
 360
 361         /* Ignoring cache creation failure */
 362         c = auxtrace_cache__new(bits, sizeof(struct intel_pt_cache_entry), 200);
 363
 364         dso->auxtrace_cache = c;
 365
 366         return c;
 367 }
 368
 369 static int intel_pt_cache_add(struct dso *dso, struct machine *machine,
 370                               u64 offset, u64 insn_cnt, u64 byte_cnt,
 371                               struct intel_pt_insn *intel_pt_insn)
 372 {
 373         struct auxtrace_cache *c = intel_pt_cache(dso, machine);
 374         struct intel_pt_cache_entry *e;
 375         int err;
 376
 377         if (!c)
 378                 return -ENOMEM;
 379
 380         e = auxtrace_cache__alloc_entry(c);
 381         if (!e)
 382                 return -ENOMEM;
 383
 384         e->insn_cnt = insn_cnt;
 385         e->byte_cnt = byte_cnt;
 386         e->op = intel_pt_insn->op;
 387         e->branch = intel_pt_insn->branch;
 388         e->length = intel_pt_insn->length;
 389         e->rel = intel_pt_insn->rel;
 390
 391         err = auxtrace_cache__add(c, offset, &e->entry);
 392         if (err)
 393                 auxtrace_cache__free_entry(c, e);
 394
 395         return err;
 396 }
 397
 398 static struct intel_pt_cache_entry *
 399 intel_pt_cache_lookup(struct dso *dso, struct machine *machine, u64 offset)
 400 {
 401         struct auxtrace_cache *c = intel_pt_cache(dso, machine);
 402
 403         if (!c)
 404                 return NULL;
 405
 406         return auxtrace_cache__lookup(dso->auxtrace_cache, offset);
 407 }
 408
 409 static int intel_pt_walk_next_insn(struct intel_pt_insn *intel_pt_insn,
 410                                    uint64_t *insn_cnt_ptr, uint64_t *ip,
 411                                    uint64_t to_ip, uint64_t max_insn_cnt,
 412                                    void *data)
 413 {
 414         struct intel_pt_queue *ptq = data;
 415         struct machine *machine = ptq->pt->machine;
 416         struct thread *thread;
 417         struct addr_location al;
 418         unsigned char buf[1024];
 419         size_t bufsz;
 420         ssize_t len;
 421         int x86_64;
 422         u8 cpumode;
 423         u64 offset, start_offset, start_ip;
 424         u64 insn_cnt = 0;
 425         bool one_map = true;
 426
 427         if (to_ip && *ip == to_ip)
 428                 goto out_no_cache;
 429
 430         bufsz = intel_pt_insn_max_size();
 431
 432         if (*ip >= ptq->pt->kernel_start)
 433                 cpumode = PERF_RECORD_MISC_KERNEL;
 434         else
 435                 cpumode = PERF_RECORD_MISC_USER;
 436
 437         thread = ptq->thread;
 438         if (!thread) {
 439                 if (cpumode != PERF_RECORD_MISC_KERNEL)
 440                         return -EINVAL;
 441                 thread = ptq->pt->unknown_thread;
 442         }
 443
 444         while (1) {
 445                 thread__find_addr_map(thread, cpumode, MAP__FUNCTION, *ip, &al);
 446                 if (!al.map || !al.map->dso)
 447                         return -EINVAL;
 448
 449                 if (al.map->dso->data.status == DSO_DATA_STATUS_ERROR &&
 450                     dso__data_status_seen(al.map->dso,
 451                                           DSO_DATA_STATUS_SEEN_ITRACE))
 452                         return -ENOENT;
 453
 454                 offset = al.map->map_ip(al.map, *ip);
 455
 456                 if (!to_ip && one_map) {
 457                         struct intel_pt_cache_entry *e;
 458
 459                         e = intel_pt_cache_lookup(al.map->dso, machine, offset);
 460                         if (e &&
 461                             (!max_insn_cnt || e->insn_cnt <= max_insn_cnt)) {
 462                                 *insn_cnt_ptr = e->insn_cnt;
 463                                 *ip += e->byte_cnt;
 464                                 intel_pt_insn->op = e->op;
 465                                 intel_pt_insn->branch = e->branch;
 466                                 intel_pt_insn->length = e->length;
 467                                 intel_pt_insn->rel = e->rel;
 468                                 intel_pt_log_insn_no_data(intel_pt_insn, *ip);
 469                                 return 0;
 470                         }
 471                 }
 472
 473                 start_offset = offset;
 474                 start_ip = *ip;
 475
 476                 /* Load maps to ensure dso->is_64_bit has been updated */
 477                 map__load(al.map, machine->symbol_filter);
 478
 479                 x86_64 = al.map->dso->is_64_bit;
 480
 481                 while (1) {
 482                         len = dso__data_read_offset(al.map->dso, machine,
 483                                                     offset, buf, bufsz);
 484                         if (len <= 0)
 485                                 return -EINVAL;
 486
 487                         if (intel_pt_get_insn(buf, len, x86_64, intel_pt_insn))
 488                                 return -EINVAL;
 489
 490                         intel_pt_log_insn(intel_pt_insn, *ip);
 491
 492                         insn_cnt += 1;
 493
 494                         if (intel_pt_insn->branch != INTEL_PT_BR_NO_BRANCH)
 495                                 goto out;
 496
 497                         if (max_insn_cnt && insn_cnt >= max_insn_cnt)
 498                                 goto out_no_cache;
 499
 500                         *ip += intel_pt_insn->length;
 501
 502                         if (to_ip && *ip == to_ip)
 503                                 goto out_no_cache;
 504
 505                         if (*ip >= al.map->end)
 506                                 break;
 507
 508                         offset += intel_pt_insn->length;
 509                 }
 510                 one_map = false;
 511         }
 512 out:
 513         *insn_cnt_ptr = insn_cnt;
 514
 515         if (!one_map)
 516                 goto out_no_cache;
 517
 518         /*
 519          * Didn't lookup in the 'to_ip' case, so do it now to prevent duplicate
 520          * entries.
 521          */
 522         if (to_ip) {
 523                 struct intel_pt_cache_entry *e;
 524
 525                 e = intel_pt_cache_lookup(al.map->dso, machine, start_offset);
 526                 if (e)
 527                         return 0;
 528         }
 529
 530         /* Ignore cache errors */
 531         intel_pt_cache_add(al.map->dso, machine, start_offset, insn_cnt,
 532                            *ip - start_ip, intel_pt_insn);
 533
 534         return 0;
 535
 536 out_no_cache:
 537         *insn_cnt_ptr = insn_cnt;
 538         return 0;
 539 }
 540
 541 static bool intel_pt_get_config(struct intel_pt *pt,
 542                                 struct perf_event_attr *attr, u64 *config)
 543 {
 544         if (attr->type == pt->pmu_type) {
 545                 if (config)
 546                         *config = attr->config;
 547                 return true;
 548         }
 549
 550         return false;
 551 }
 552
 553 static bool intel_pt_exclude_kernel(struct intel_pt *pt)
 554 {
 555         struct perf_evsel *evsel;
 556
 557         evlist__for_each(pt->session->evlist, evsel) {
 558                 if (intel_pt_get_config(pt, &evsel->attr, NULL) &&
 559                     !evsel->attr.exclude_kernel)
 560                         return false;
 561         }
 562         return true;
 563 }
 564
 565 static bool intel_pt_return_compression(struct intel_pt *pt)
 566 {
 567         struct perf_evsel *evsel;
 568         u64 config;
 569
 570         if (!pt->noretcomp_bit)
 571                 return true;
 572
 573         evlist__for_each(pt->session->evlist, evsel) {
 574                 if (intel_pt_get_config(pt, &evsel->attr, &config) &&
 575                     (config & pt->noretcomp_bit))
 576                         return false;
 577         }
 578         return true;
 579 }
 580
 581 static unsigned int intel_pt_mtc_period(struct intel_pt *pt)
 582 {
 583         struct perf_evsel *evsel;
 584         unsigned int shift;
 585         u64 config;
 586
 587         if (!pt->mtc_freq_bits)
 588                 return 0;
 589
 590         for (shift = 0, config = pt->mtc_freq_bits; !(config & 1); shift++)
 591                 config >>= 1;
 592
 593         evlist__for_each(pt->session->evlist, evsel) {
 594                 if (intel_pt_get_config(pt, &evsel->attr, &config))
 595                         return (config & pt->mtc_freq_bits) >> shift;
 596         }
 597         return 0;
 598 }
 599
 600 static bool intel_pt_timeless_decoding(struct intel_pt *pt)
 601 {
 602         struct perf_evsel *evsel;
 603         bool timeless_decoding = true;
 604         u64 config;
 605
 606         if (!pt->tsc_bit || !pt->cap_user_time_zero)
 607                 return true;
 608
 609         evlist__for_each(pt->session->evlist, evsel) {
 610                 if (!(evsel->attr.sample_type & PERF_SAMPLE_TIME))
 611                         return true;
 612                 if (intel_pt_get_config(pt, &evsel->attr, &config)) {
 613                         if (config & pt->tsc_bit)
 614                                 timeless_decoding = false;
 615                         else
 616                                 return true;
 617                 }
 618         }
 619         return timeless_decoding;
 620 }
 621
 622 static bool intel_pt_tracing_kernel(struct intel_pt *pt)
 623 {
 624         struct perf_evsel *evsel;
 625
 626         evlist__for_each(pt->session->evlist, evsel) {
 627                 if (intel_pt_get_config(pt, &evsel->attr, NULL) &&
 628                     !evsel->attr.exclude_kernel)
 629                         return true;
 630         }
 631         return false;
 632 }
 633
 634 static bool intel_pt_have_tsc(struct intel_pt *pt)
 635 {
 636         struct perf_evsel *evsel;
 637         bool have_tsc = false;
 638         u64 config;
 639
 640         if (!pt->tsc_bit)
 641                 return false;
 642
 643         evlist__for_each(pt->session->evlist, evsel) {
 644                 if (intel_pt_get_config(pt, &evsel->attr, &config)) {
 645                         if (config & pt->tsc_bit)
 646                                 have_tsc = true;
 647                         else
 648                                 return false;
 649                 }
 650         }
 651         return have_tsc;
 652 }
 653
 654 static u64 intel_pt_ns_to_ticks(const struct intel_pt *pt, u64 ns)
 655 {
 656         u64 quot, rem;
 657
 658         quot = ns / pt->tc.time_mult;
 659         rem  = ns % pt->tc.time_mult;
 660         return (quot << pt->tc.time_shift) + (rem << pt->tc.time_shift) /
 661                 pt->tc.time_mult;
 662 }
 663
 664 static struct intel_pt_queue *intel_pt_alloc_queue(struct intel_pt *pt,
 665                                                    unsigned int queue_nr)
 666 {
 667         struct intel_pt_params params = { .get_trace = 0, };
 668         struct intel_pt_queue *ptq;
 669
 670         ptq = zalloc(sizeof(struct intel_pt_queue));
 671         if (!ptq)
 672                 return NULL;
 673
 674         if (pt->synth_opts.callchain) {
 675                 size_t sz = sizeof(struct ip_callchain);
 676
 677                 sz += pt->synth_opts.callchain_sz * sizeof(u64);
 678                 ptq->chain = zalloc(sz);
 679                 if (!ptq->chain)
 680                         goto out_free;
 681         }
 682
 683         if (pt->synth_opts.last_branch) {
 684                 size_t sz = sizeof(struct branch_stack);
 685
 686                 sz += pt->synth_opts.last_branch_sz *
 687                       sizeof(struct branch_entry);
 688                 ptq->last_branch = zalloc(sz);
 689                 if (!ptq->last_branch)
 690                         goto out_free;
 691                 ptq->last_branch_rb = zalloc(sz);
 692                 if (!ptq->last_branch_rb)
 693                         goto out_free;
 694         }
 695
 696         ptq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE);
 697         if (!ptq->event_buf)
 698                 goto out_free;
 699
 700         ptq->pt = pt;
 701         ptq->queue_nr = queue_nr;
 702         ptq->exclude_kernel = intel_pt_exclude_kernel(pt);
 703         ptq->pid = -1;
 704         ptq->tid = -1;
 705         ptq->cpu = -1;
 706         ptq->next_tid = -1;
 707
 708         params.get_trace = intel_pt_get_trace;
 709         params.walk_insn = intel_pt_walk_next_insn;
 710         params.data = ptq;
 711         params.return_compression = intel_pt_return_compression(pt);
 712         params.max_non_turbo_ratio = pt->max_non_turbo_ratio;
 713         params.mtc_period = intel_pt_mtc_period(pt);
 714         params.tsc_ctc_ratio_n = pt->tsc_ctc_ratio_n;
 715         params.tsc_ctc_ratio_d = pt->tsc_ctc_ratio_d;
 716
 717         if (pt->synth_opts.instructions) {
 718                 if (pt->synth_opts.period) {
 719                         switch (pt->synth_opts.period_type) {
 720                         case PERF_ITRACE_PERIOD_INSTRUCTIONS:
 721                                 params.period_type =
 722                                                 INTEL_PT_PERIOD_INSTRUCTIONS;
 723                                 params.period = pt->synth_opts.period;
 724                                 break;
 725                         case PERF_ITRACE_PERIOD_TICKS:
 726                                 params.period_type = INTEL_PT_PERIOD_TICKS;
 727                                 params.period = pt->synth_opts.period;
 728                                 break;
 729                         case PERF_ITRACE_PERIOD_NANOSECS:
 730                                 params.period_type = INTEL_PT_PERIOD_TICKS;
 731                                 params.period = intel_pt_ns_to_ticks(pt,
 732                                                         pt->synth_opts.period);
 733                                 break;
 734                         default:
 735                                 break;
 736                         }
 737                 }
 738
 739                 if (!params.period) {
 740                         params.period_type = INTEL_PT_PERIOD_INSTRUCTIONS;
 741                         params.period = 1;
 742                 }
 743         }
 744
 745         ptq->decoder = intel_pt_decoder_new(&params);
 746         if (!ptq->decoder)
 747                 goto out_free;
 748
 749         return ptq;
 750
 751 out_free:
 752         zfree(&ptq->event_buf);
 753         zfree(&ptq->last_branch);
 754         zfree(&ptq->last_branch_rb);
 755         zfree(&ptq->chain);
 756         free(ptq);
 757         return NULL;
 758 }
 759
 760 static void intel_pt_free_queue(void *priv)
 761 {
 762         struct intel_pt_queue *ptq = priv;
 763
 764         if (!ptq)
 765                 return;
 766         thread__zput(ptq->thread);
 767         intel_pt_decoder_free(ptq->decoder);
 768         zfree(&ptq->event_buf);
 769         zfree(&ptq->last_branch);
 770         zfree(&ptq->last_branch_rb);
 771         zfree(&ptq->chain);
 772         free(ptq);
 773 }
 774
 775 static void intel_pt_set_pid_tid_cpu(struct intel_pt *pt,
 776                                      struct auxtrace_queue *queue)
 777 {
 778         struct intel_pt_queue *ptq = queue->priv;
 779
 780         if (queue->tid == -1 || pt->have_sched_switch) {
 781                 ptq->tid = machine__get_current_tid(pt->machine, ptq->cpu);
 782                 thread__zput(ptq->thread);
 783         }
 784
 785         if (!ptq->thread && ptq->tid != -1)
 786                 ptq->thread = machine__find_thread(pt->machine, -1, ptq->tid);
 787
 788         if (ptq->thread) {
 789                 ptq->pid = ptq->thread->pid_;
 790                 if (queue->cpu == -1)
 791                         ptq->cpu = ptq->thread->cpu;
 792         }
 793 }
 794
 795 static void intel_pt_sample_flags(struct intel_pt_queue *ptq)
 796 {
 797         if (ptq->state->flags & INTEL_PT_ABORT_TX) {
 798                 ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_TX_ABORT;
 799         } else if (ptq->state->flags & INTEL_PT_ASYNC) {
 800                 if (ptq->state->to_ip)
 801                         ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL |
 802                                      PERF_IP_FLAG_ASYNC |
 803                                      PERF_IP_FLAG_INTERRUPT;
 804                 else
 805                         ptq->flags = PERF_IP_FLAG_BRANCH |
 806                                      PERF_IP_FLAG_TRACE_END;
 807                 ptq->insn_len = 0;
 808         } else {
 809                 if (ptq->state->from_ip)
 810                         ptq->flags = intel_pt_insn_type(ptq->state->insn_op);
 811                 else
 812                         ptq->flags = PERF_IP_FLAG_BRANCH |
 813                                      PERF_IP_FLAG_TRACE_BEGIN;
 814                 if (ptq->state->flags & INTEL_PT_IN_TX)
 815                         ptq->flags |= PERF_IP_FLAG_IN_TX;
 816                 ptq->insn_len = ptq->state->insn_len;
 817         }
 818 }
 819
 820 static int intel_pt_setup_queue(struct intel_pt *pt,
 821                                 struct auxtrace_queue *queue,
 822                                 unsigned int queue_nr)
 823 {
 824         struct intel_pt_queue *ptq = queue->priv;
 825
 826         if (list_empty(&queue->head))
 827                 return 0;
 828
 829         if (!ptq) {
 830                 ptq = intel_pt_alloc_queue(pt, queue_nr);
 831                 if (!ptq)
 832                         return -ENOMEM;
 833                 queue->priv = ptq;
 834
 835                 if (queue->cpu != -1)
 836                         ptq->cpu = queue->cpu;
 837                 ptq->tid = queue->tid;
 838
 839                 if (pt->sampling_mode) {
 840                         if (pt->timeless_decoding)
 841                                 ptq->step_through_buffers = true;
 842                         if (pt->timeless_decoding || !pt->have_sched_switch)
 843                                 ptq->use_buffer_pid_tid = true;
 844                 }
 845         }
 846
 847         if (!ptq->on_heap &&
 848             (!pt->sync_switch ||
 849              ptq->switch_state != INTEL_PT_SS_EXPECTING_SWITCH_EVENT)) {
 850                 const struct intel_pt_state *state;
 851                 int ret;
 852
 853                 if (pt->timeless_decoding)
 854                         return 0;
 855
 856                 intel_pt_log("queue %u getting timestamp\n", queue_nr);
 857                 intel_pt_log("queue %u decoding cpu %d pid %d tid %d\n",
 858                              queue_nr, ptq->cpu, ptq->pid, ptq->tid);
 859                 while (1) {
 860                         state = intel_pt_decode(ptq->decoder);
 861                         if (state->err) {
 862                                 if (state->err == INTEL_PT_ERR_NODATA) {
 863                                         intel_pt_log("queue %u has no timestamp\n",
 864                                                      queue_nr);
 865                                         return 0;
 866                                 }
 867                                 continue;
 868                         }
 869                         if (state->timestamp)
 870                                 break;
 871                 }
 872
 873                 ptq->timestamp = state->timestamp;
 874                 intel_pt_log("queue %u timestamp 0x%" PRIx64 "\n",
 875                              queue_nr, ptq->timestamp);
 876                 ptq->state = state;
 877                 ptq->have_sample = true;
 878                 intel_pt_sample_flags(ptq);
 879                 ret = auxtrace_heap__add(&pt->heap, queue_nr, ptq->timestamp);
 880                 if (ret)
 881                         return ret;
 882                 ptq->on_heap = true;
 883         }
 884
 885         return 0;
 886 }
 887
 888 static int intel_pt_setup_queues(struct intel_pt *pt)
 889 {
 890         unsigned int i;
 891         int ret;
 892
 893         for (i = 0; i < pt->queues.nr_queues; i++) {
 894                 ret = intel_pt_setup_queue(pt, &pt->queues.queue_array[i], i);
 895                 if (ret)
 896                         return ret;
 897         }
 898         return 0;
 899 }
 900
 901 static inline void intel_pt_copy_last_branch_rb(struct intel_pt_queue *ptq)
 902 {
 903         struct branch_stack *bs_src = ptq->last_branch_rb;
 904         struct branch_stack *bs_dst = ptq->last_branch;
 905         size_t nr = 0;
 906
 907         bs_dst->nr = bs_src->nr;
 908
 909         if (!bs_src->nr)
 910                 return;
 911
 912         nr = ptq->pt->synth_opts.last_branch_sz - ptq->last_branch_pos;
 913         memcpy(&bs_dst->entries[0],
 914                &bs_src->entries[ptq->last_branch_pos],
 915                sizeof(struct branch_entry) * nr);
 916
 917         if (bs_src->nr >= ptq->pt->synth_opts.last_branch_sz) {
 918                 memcpy(&bs_dst->entries[nr],
 919                        &bs_src->entries[0],
 920                        sizeof(struct branch_entry) * ptq->last_branch_pos);
 921         }
 922 }
 923
 924 static inline void intel_pt_reset_last_branch_rb(struct intel_pt_queue *ptq)
 925 {
 926         ptq->last_branch_pos = 0;
 927         ptq->last_branch_rb->nr = 0;
 928 }
 929
 930 static void intel_pt_update_last_branch_rb(struct intel_pt_queue *ptq)
 931 {
 932         const struct intel_pt_state *state = ptq->state;
 933         struct branch_stack *bs = ptq->last_branch_rb;
 934         struct branch_entry *be;
 935
 936         if (!ptq->last_branch_pos)
 937                 ptq->last_branch_pos = ptq->pt->synth_opts.last_branch_sz;
 938
 939         ptq->last_branch_pos -= 1;
 940
 941         be              = &bs->entries[ptq->last_branch_pos];
 942         be->from        = state->from_ip;
 943         be->to          = state->to_ip;
 944         be->flags.abort = !!(state->flags & INTEL_PT_ABORT_TX);
 945         be->flags.in_tx = !!(state->flags & INTEL_PT_IN_TX);
 946         /* No support for mispredict */
 947         be->flags.mispred = ptq->pt->mispred_all;
 948
 949         if (bs->nr < ptq->pt->synth_opts.last_branch_sz)
 950                 bs->nr += 1;
 951 }
 952
 953 static int intel_pt_inject_event(union perf_event *event,
 954                                  struct perf_sample *sample, u64 type,
 955                                  bool swapped)
 956 {
 957         event->header.size = perf_event__sample_event_size(sample, type, 0);
 958         return perf_event__synthesize_sample(event, type, 0, sample, swapped);
 959 }
 960
 961 static int intel_pt_synth_branch_sample(struct intel_pt_queue *ptq)
 962 {
 963         int ret;
 964         struct intel_pt *pt = ptq->pt;
 965         union perf_event *event = ptq->event_buf;
 966         struct perf_sample sample = { .ip = 0, };
 967         struct dummy_branch_stack {
 968                 u64                     nr;
 969                 struct branch_entry     entries;
 970         } dummy_bs;
 971
 972         if (pt->branches_filter && !(pt->branches_filter & ptq->flags))
 973                 return 0;
 974
 975         event->sample.header.type = PERF_RECORD_SAMPLE;
 976         event->sample.header.misc = PERF_RECORD_MISC_USER;
 977         event->sample.header.size = sizeof(struct perf_event_header);
 978
 979         if (!pt->timeless_decoding)
 980                 sample.time = tsc_to_perf_time(ptq->timestamp, &pt->tc);
 981
 982         sample.ip = ptq->state->from_ip;
 983         sample.pid = ptq->pid;
 984         sample.tid = ptq->tid;
 985         sample.addr = ptq->state->to_ip;
 986         sample.id = ptq->pt->branches_id;
 987         sample.stream_id = ptq->pt->branches_id;
 988         sample.period = 1;
 989         sample.cpu = ptq->cpu;
 990         sample.flags = ptq->flags;
 991         sample.insn_len = ptq->insn_len;
 992
 993         /*
 994          * perf report cannot handle events without a branch stack when using
 995          * SORT_MODE__BRANCH so make a dummy one.
 996          */
 997         if (pt->synth_opts.last_branch && sort__mode == SORT_MODE__BRANCH) {
 998                 dummy_bs = (struct dummy_branch_stack){
 999                         .nr = 1,
1000                         .entries = {
1001                                 .from = sample.ip,
1002                                 .to = sample.addr,
1003                         },
1004                 };
1005                 sample.branch_stack = (struct branch_stack *)&dummy_bs;
1006         }
1007
1008         if (pt->synth_opts.inject) {
1009                 ret = intel_pt_inject_event(event, &sample,
1010                                             pt->branches_sample_type,
1011                                             pt->synth_needs_swap);
1012                 if (ret)
1013                         return ret;
1014         }
1015
1016         ret = perf_session__deliver_synth_event(pt->session, event, &sample);
1017         if (ret)
1018                 pr_err("Intel Processor Trace: failed to deliver branch event, error %d\n",
1019                        ret);
1020
1021         return ret;
1022 }
1023
1024 static int intel_pt_synth_instruction_sample(struct intel_pt_queue *ptq)
1025 {
1026         int ret;
1027         struct intel_pt *pt = ptq->pt;
1028         union perf_event *event = ptq->event_buf;
1029         struct perf_sample sample = { .ip = 0, };
1030
1031         event->sample.header.type = PERF_RECORD_SAMPLE;
1032         event->sample.header.misc = PERF_RECORD_MISC_USER;
1033         event->sample.header.size = sizeof(struct perf_event_header);
1034
1035         if (!pt->timeless_decoding)
1036                 sample.time = tsc_to_perf_time(ptq->timestamp, &pt->tc);
1037
1038         sample.ip = ptq->state->from_ip;
1039         sample.pid = ptq->pid;
1040         sample.tid = ptq->tid;
1041         sample.addr = ptq->state->to_ip;
1042         sample.id = ptq->pt->instructions_id;
1043         sample.stream_id = ptq->pt->instructions_id;
1044         sample.period = ptq->state->tot_insn_cnt - ptq->last_insn_cnt;
1045         sample.cpu = ptq->cpu;
1046         sample.flags = ptq->flags;
1047         sample.insn_len = ptq->insn_len;
1048
1049         ptq->last_insn_cnt = ptq->state->tot_insn_cnt;
1050
1051         if (pt->synth_opts.callchain) {
1052                 thread_stack__sample(ptq->thread, ptq->chain,
1053                                      pt->synth_opts.callchain_sz, sample.ip);
1054                 sample.callchain = ptq->chain;
1055         }
1056
1057         if (pt->synth_opts.last_branch) {
1058                 intel_pt_copy_last_branch_rb(ptq);
1059                 sample.branch_stack = ptq->last_branch;
1060         }
1061
1062         if (pt->synth_opts.inject) {
1063                 ret = intel_pt_inject_event(event, &sample,
1064                                             pt->instructions_sample_type,
1065                                             pt->synth_needs_swap);
1066                 if (ret)
1067                         return ret;
1068         }
1069
1070         ret = perf_session__deliver_synth_event(pt->session, event, &sample);
1071         if (ret)
1072                 pr_err("Intel Processor Trace: failed to deliver instruction event, error %d\n",
1073                        ret);
1074
1075         if (pt->synth_opts.last_branch)
1076                 intel_pt_reset_last_branch_rb(ptq);
1077
1078         return ret;
1079 }
1080
1081 static int intel_pt_synth_transaction_sample(struct intel_pt_queue *ptq)
1082 {
1083         int ret;
1084         struct intel_pt *pt = ptq->pt;
1085         union perf_event *event = ptq->event_buf;
1086         struct perf_sample sample = { .ip = 0, };
1087
1088         event->sample.header.type = PERF_RECORD_SAMPLE;
1089         event->sample.header.misc = PERF_RECORD_MISC_USER;
1090         event->sample.header.size = sizeof(struct perf_event_header);
1091
1092         if (!pt->timeless_decoding)
1093                 sample.time = tsc_to_perf_time(ptq->timestamp, &pt->tc);
1094
1095         sample.ip = ptq->state->from_ip;
1096         sample.pid = ptq->pid;
1097         sample.tid = ptq->tid;
1098         sample.addr = ptq->state->to_ip;
1099         sample.id = ptq->pt->transactions_id;
1100         sample.stream_id = ptq->pt->transactions_id;
1101         sample.period = 1;
1102         sample.cpu = ptq->cpu;
1103         sample.flags = ptq->flags;
1104         sample.insn_len = ptq->insn_len;
1105
1106         if (pt->synth_opts.callchain) {
1107                 thread_stack__sample(ptq->thread, ptq->chain,
1108                                      pt->synth_opts.callchain_sz, sample.ip);
1109                 sample.callchain = ptq->chain;
1110         }
1111
1112         if (pt->synth_opts.last_branch) {
1113                 intel_pt_copy_last_branch_rb(ptq);
1114                 sample.branch_stack = ptq->last_branch;
1115         }
1116
1117         if (pt->synth_opts.inject) {
1118                 ret = intel_pt_inject_event(event, &sample,
1119                                             pt->transactions_sample_type,
1120                                             pt->synth_needs_swap);
1121                 if (ret)
1122                         return ret;
1123         }
1124
1125         ret = perf_session__deliver_synth_event(pt->session, event, &sample);
1126         if (ret)
1127                 pr_err("Intel Processor Trace: failed to deliver transaction event, error %d\n",
1128                        ret);
1129
1130         if (pt->synth_opts.callchain)
1131                 intel_pt_reset_last_branch_rb(ptq);
1132
1133         return ret;
1134 }
1135
1136 static int intel_pt_synth_error(struct intel_pt *pt, int code, int cpu,
1137                                 pid_t pid, pid_t tid, u64 ip)
1138 {
1139         union perf_event event;
1140         char msg[MAX_AUXTRACE_ERROR_MSG];
1141         int err;
1142
1143         intel_pt__strerror(code, msg, MAX_AUXTRACE_ERROR_MSG);
1144
1145         auxtrace_synth_error(&event.auxtrace_error, PERF_AUXTRACE_ERROR_ITRACE,
1146                              code, cpu, pid, tid, ip, msg);
1147
1148         err = perf_session__deliver_synth_event(pt->session, &event, NULL);
1149         if (err)
1150                 pr_err("Intel Processor Trace: failed to deliver error event, error %d\n",
1151                        err);
1152
1153         return err;
1154 }
1155
1156 static int intel_pt_next_tid(struct intel_pt *pt, struct intel_pt_queue *ptq)
1157 {
1158         struct auxtrace_queue *queue;
1159         pid_t tid = ptq->next_tid;
1160         int err;
1161
1162         if (tid == -1)
1163                 return 0;
1164
1165         intel_pt_log("switch: cpu %d tid %d\n", ptq->cpu, tid);
1166
1167         err = machine__set_current_tid(pt->machine, ptq->cpu, -1, tid);
1168
1169         queue = &pt->queues.queue_array[ptq->queue_nr];
1170         intel_pt_set_pid_tid_cpu(pt, queue);
1171
1172         ptq->next_tid = -1;
1173
1174         return err;
1175 }
1176
1177 static inline bool intel_pt_is_switch_ip(struct intel_pt_queue *ptq, u64 ip)
1178 {
1179         struct intel_pt *pt = ptq->pt;
1180
1181         return ip == pt->switch_ip &&
1182                (ptq->flags & PERF_IP_FLAG_BRANCH) &&
1183                !(ptq->flags & (PERF_IP_FLAG_CONDITIONAL | PERF_IP_FLAG_ASYNC |
1184                                PERF_IP_FLAG_INTERRUPT | PERF_IP_FLAG_TX_ABORT));
1185 }
1186
1187 static int intel_pt_sample(struct intel_pt_queue *ptq)
1188 {
1189         const struct intel_pt_state *state = ptq->state;
1190         struct intel_pt *pt = ptq->pt;
1191         int err;
1192
1193         if (!ptq->have_sample)
1194                 return 0;
1195
1196         ptq->have_sample = false;
1197
1198         if (pt->sample_instructions &&
1199             (state->type & INTEL_PT_INSTRUCTION)) {
1200                 err = intel_pt_synth_instruction_sample(ptq);
1201                 if (err)
1202                         return err;
1203         }
1204
1205         if (pt->sample_transactions &&
1206             (state->type & INTEL_PT_TRANSACTION)) {
1207                 err = intel_pt_synth_transaction_sample(ptq);
1208                 if (err)
1209                         return err;
1210         }
1211
1212         if (!(state->type & INTEL_PT_BRANCH))
1213                 return 0;
1214
1215         if (pt->synth_opts.callchain)
1216                 thread_stack__event(ptq->thread, ptq->flags, state->from_ip,
1217                                     state->to_ip, ptq->insn_len,
1218                                     state->trace_nr);
1219         else
1220                 thread_stack__set_trace_nr(ptq->thread, state->trace_nr);
1221
1222         if (pt->sample_branches) {
1223                 err = intel_pt_synth_branch_sample(ptq);
1224                 if (err)
1225                         return err;
1226         }
1227
1228         if (pt->synth_opts.last_branch)
1229                 intel_pt_update_last_branch_rb(ptq);
1230
1231         if (!pt->sync_switch)
1232                 return 0;
1233
1234         if (intel_pt_is_switch_ip(ptq, state->to_ip)) {
1235                 switch (ptq->switch_state) {
1236                 case INTEL_PT_SS_UNKNOWN:
1237                 case INTEL_PT_SS_EXPECTING_SWITCH_IP:
1238                         err = intel_pt_next_tid(pt, ptq);
1239                         if (err)
1240                                 return err;
1241                         ptq->switch_state = INTEL_PT_SS_TRACING;
1242                         break;
1243                 default:
1244                         ptq->switch_state = INTEL_PT_SS_EXPECTING_SWITCH_EVENT;
1245                         return 1;
1246                 }
1247         } else if (!state->to_ip) {
1248                 ptq->switch_state = INTEL_PT_SS_NOT_TRACING;
1249         } else if (ptq->switch_state == INTEL_PT_SS_NOT_TRACING) {
1250                 ptq->switch_state = INTEL_PT_SS_UNKNOWN;
1251         } else if (ptq->switch_state == INTEL_PT_SS_UNKNOWN &&
1252                    state->to_ip == pt->ptss_ip &&
1253                    (ptq->flags & PERF_IP_FLAG_CALL)) {
1254                 ptq->switch_state = INTEL_PT_SS_TRACING;
1255         }
1256
1257         return 0;
1258 }
1259
1260 static u64 intel_pt_switch_ip(struct intel_pt *pt, u64 *ptss_ip)
1261 {
1262         struct machine *machine = pt->machine;
1263         struct map *map;
1264         struct symbol *sym, *start;
1265         u64 ip, switch_ip = 0;
1266         const char *ptss;
1267
1268         if (ptss_ip)
1269                 *ptss_ip = 0;
1270
1271         map = machine__kernel_map(machine);
1272         if (!map)
1273                 return 0;
1274
1275         if (map__load(map, machine->symbol_filter))
1276                 return 0;
1277
1278         start = dso__first_symbol(map->dso, MAP__FUNCTION);
1279
1280         for (sym = start; sym; sym = dso__next_symbol(sym)) {
1281                 if (sym->binding == STB_GLOBAL &&
1282                     !strcmp(sym->name, "__switch_to")) {
1283                         ip = map->unmap_ip(map, sym->start);
1284                         if (ip >= map->start && ip < map->end) {
1285                                 switch_ip = ip;
1286                                 break;
1287                         }
1288                 }
1289         }
1290
1291         if (!switch_ip || !ptss_ip)
1292                 return 0;
1293
1294         if (pt->have_sched_switch == 1)
1295                 ptss = "perf_trace_sched_switch";
1296         else
1297                 ptss = "__perf_event_task_sched_out";
1298
1299         for (sym = start; sym; sym = dso__next_symbol(sym)) {
1300                 if (!strcmp(sym->name, ptss)) {
1301                         ip = map->unmap_ip(map, sym->start);
1302                         if (ip >= map->start && ip < map->end) {
1303                                 *ptss_ip = ip;
1304                                 break;
1305                         }
1306                 }
1307         }
1308
1309         return switch_ip;
1310 }
1311
1312 static int intel_pt_run_decoder(struct intel_pt_queue *ptq, u64 *timestamp)
1313 {
1314         const struct intel_pt_state *state = ptq->state;
1315         struct intel_pt *pt = ptq->pt;
1316         int err;
1317
1318         if (!pt->kernel_start) {
1319                 pt->kernel_start = machine__kernel_start(pt->machine);
1320                 if (pt->per_cpu_mmaps &&
1321                     (pt->have_sched_switch == 1 || pt->have_sched_switch == 3) &&
1322                     !pt->timeless_decoding && intel_pt_tracing_kernel(pt) &&
1323                     !pt->sampling_mode) {
1324                         pt->switch_ip = intel_pt_switch_ip(pt, &pt->ptss_ip);
1325                         if (pt->switch_ip) {
1326                                 intel_pt_log("switch_ip: %"PRIx64" ptss_ip: %"PRIx64"\n",
1327                                              pt->switch_ip, pt->ptss_ip);
1328                                 pt->sync_switch = true;
1329                         }
1330                 }
1331         }
1332
1333         intel_pt_log("queue %u decoding cpu %d pid %d tid %d\n",
1334                      ptq->queue_nr, ptq->cpu, ptq->pid, ptq->tid);
1335         while (1) {
1336                 err = intel_pt_sample(ptq);
1337                 if (err)
1338                         return err;
1339
1340                 state = intel_pt_decode(ptq->decoder);
1341                 if (state->err) {
1342                         if (state->err == INTEL_PT_ERR_NODATA)
1343                                 return 1;
1344                         if (pt->sync_switch &&
1345                             state->from_ip >= pt->kernel_start) {
1346                                 pt->sync_switch = false;
1347                                 intel_pt_next_tid(pt, ptq);
1348                         }
1349                         if (pt->synth_opts.errors) {
1350                                 err = intel_pt_synth_error(pt, state->err,
1351                                                            ptq->cpu, ptq->pid,
1352                                                            ptq->tid,
1353                                                            state->from_ip);
1354                                 if (err)
1355                                         return err;
1356                         }
1357                         continue;
1358                 }
1359
1360                 ptq->state = state;
1361                 ptq->have_sample = true;
1362                 intel_pt_sample_flags(ptq);
1363
1364                 /* Use estimated TSC upon return to user space */
1365                 if (pt->est_tsc &&
1366                     (state->from_ip >= pt->kernel_start || !state->from_ip) &&
1367                     state->to_ip && state->to_ip < pt->kernel_start) {
1368                         intel_pt_log("TSC %"PRIx64" est. TSC %"PRIx64"\n",
1369                                      state->timestamp, state->est_timestamp);
1370                         ptq->timestamp = state->est_timestamp;
1371                 /* Use estimated TSC in unknown switch state */
1372                 } else if (pt->sync_switch &&
1373                            ptq->switch_state == INTEL_PT_SS_UNKNOWN &&
1374                            intel_pt_is_switch_ip(ptq, state->to_ip) &&
1375                            ptq->next_tid == -1) {
1376                         intel_pt_log("TSC %"PRIx64" est. TSC %"PRIx64"\n",
1377                                      state->timestamp, state->est_timestamp);
1378                         ptq->timestamp = state->est_timestamp;
1379                 } else if (state->timestamp > ptq->timestamp) {
1380                         ptq->timestamp = state->timestamp;
1381                 }
1382
1383                 if (!pt->timeless_decoding && ptq->timestamp >= *timestamp) {
1384                         *timestamp = ptq->timestamp;
1385                         return 0;
1386                 }
1387         }
1388         return 0;
1389 }
1390
1391 static inline int intel_pt_update_queues(struct intel_pt *pt)
1392 {
1393         if (pt->queues.new_data) {
1394                 pt->queues.new_data = false;
1395                 return intel_pt_setup_queues(pt);
1396         }
1397         return 0;
1398 }
1399
1400 static int intel_pt_process_queues(struct intel_pt *pt, u64 timestamp)
1401 {
1402         unsigned int queue_nr;
1403         u64 ts;
1404         int ret;
1405
1406         while (1) {
1407                 struct auxtrace_queue *queue;
1408                 struct intel_pt_queue *ptq;
1409
1410                 if (!pt->heap.heap_cnt)
1411                         return 0;
1412
1413                 if (pt->heap.heap_array[0].ordinal >= timestamp)
1414                         return 0;
1415
1416                 queue_nr = pt->heap.heap_array[0].queue_nr;
1417                 queue = &pt->queues.queue_array[queue_nr];
1418                 ptq = queue->priv;
1419
1420                 intel_pt_log("queue %u processing 0x%" PRIx64 " to 0x%" PRIx64 "\n",
1421                              queue_nr, pt->heap.heap_array[0].ordinal,
1422                              timestamp);
1423
1424                 auxtrace_heap__pop(&pt->heap);
1425
1426                 if (pt->heap.heap_cnt) {
1427                         ts = pt->heap.heap_array[0].ordinal + 1;
1428                         if (ts > timestamp)
1429                                 ts = timestamp;
1430                 } else {
1431                         ts = timestamp;
1432                 }
1433
1434                 intel_pt_set_pid_tid_cpu(pt, queue);
1435
1436                 ret = intel_pt_run_decoder(ptq, &ts);
1437
1438                 if (ret < 0) {
1439                         auxtrace_heap__add(&pt->heap, queue_nr, ts);
1440                         return ret;
1441                 }
1442
1443                 if (!ret) {
1444                         ret = auxtrace_heap__add(&pt->heap, queue_nr, ts);
1445                         if (ret < 0)
1446                                 return ret;
1447                 } else {
1448                         ptq->on_heap = false;
1449                 }
1450         }
1451
1452         return 0;
1453 }
1454
1455 static int intel_pt_process_timeless_queues(struct intel_pt *pt, pid_t tid,
1456                                             u64 time_)
1457 {
1458         struct auxtrace_queues *queues = &pt->queues;
1459         unsigned int i;
1460         u64 ts = 0;
1461
1462         for (i = 0; i < queues->nr_queues; i++) {
1463                 struct auxtrace_queue *queue = &pt->queues.queue_array[i];
1464                 struct intel_pt_queue *ptq = queue->priv;
1465
1466                 if (ptq && (tid == -1 || ptq->tid == tid)) {
1467                         ptq->time = time_;
1468                         intel_pt_set_pid_tid_cpu(pt, queue);
1469                         intel_pt_run_decoder(ptq, &ts);
1470                 }
1471         }
1472         return 0;
1473 }
1474
1475 static int intel_pt_lost(struct intel_pt *pt, struct perf_sample *sample)
1476 {
1477         return intel_pt_synth_error(pt, INTEL_PT_ERR_LOST, sample->cpu,
1478                                     sample->pid, sample->tid, 0);
1479 }
1480
1481 static struct intel_pt_queue *intel_pt_cpu_to_ptq(struct intel_pt *pt, int cpu)
1482 {
1483         unsigned i, j;
1484
1485         if (cpu < 0 || !pt->queues.nr_queues)
1486                 return NULL;
1487
1488         if ((unsigned)cpu >= pt->queues.nr_queues)
1489                 i = pt->queues.nr_queues - 1;
1490         else
1491                 i = cpu;
1492
1493         if (pt->queues.queue_array[i].cpu == cpu)
1494                 return pt->queues.queue_array[i].priv;
1495
1496         for (j = 0; i > 0; j++) {
1497                 if (pt->queues.queue_array[--i].cpu == cpu)
1498                         return pt->queues.queue_array[i].priv;
1499         }
1500
1501         for (; j < pt->queues.nr_queues; j++) {
1502                 if (pt->queues.queue_array[j].cpu == cpu)
1503                         return pt->queues.queue_array[j].priv;
1504         }
1505
1506         return NULL;
1507 }
1508
1509 static int intel_pt_sync_switch(struct intel_pt *pt, int cpu, pid_t tid,
1510                                 u64 timestamp)
1511 {
1512         struct intel_pt_queue *ptq;
1513         int err;
1514
1515         if (!pt->sync_switch)
1516                 return 1;
1517
1518         ptq = intel_pt_cpu_to_ptq(pt, cpu);
1519         if (!ptq)
1520                 return 1;
1521
1522         switch (ptq->switch_state) {
1523         case INTEL_PT_SS_NOT_TRACING:
1524                 ptq->next_tid = -1;
1525                 break;
1526         case INTEL_PT_SS_UNKNOWN:
1527         case INTEL_PT_SS_TRACING:
1528                 ptq->next_tid = tid;
1529                 ptq->switch_state = INTEL_PT_SS_EXPECTING_SWITCH_IP;
1530                 return 0;
1531         case INTEL_PT_SS_EXPECTING_SWITCH_EVENT:
1532                 if (!ptq->on_heap) {
1533                         ptq->timestamp = perf_time_to_tsc(timestamp,
1534                                                           &pt->tc);
1535                         err = auxtrace_heap__add(&pt->heap, ptq->queue_nr,
1536                                                  ptq->timestamp);
1537                         if (err)
1538                                 return err;
1539                         ptq->on_heap = true;
1540                 }
1541                 ptq->switch_state = INTEL_PT_SS_TRACING;
1542                 break;
1543         case INTEL_PT_SS_EXPECTING_SWITCH_IP:
1544                 ptq->next_tid = tid;
1545                 intel_pt_log("ERROR: cpu %d expecting switch ip\n", cpu);
1546                 break;
1547         default:
1548                 break;
1549         }
1550
1551         return 1;
1552 }
1553
1554 static int intel_pt_process_switch(struct intel_pt *pt,
1555                                    struct perf_sample *sample)
1556 {
1557         struct perf_evsel *evsel;
1558         pid_t tid;
1559         int cpu, ret;
1560
1561         evsel = perf_evlist__id2evsel(pt->session->evlist, sample->id);
1562         if (evsel != pt->switch_evsel)
1563                 return 0;
1564
1565         tid = perf_evsel__intval(evsel, sample, "next_pid");
1566         cpu = sample->cpu;
1567
1568         intel_pt_log("sched_switch: cpu %d tid %d time %"PRIu64" tsc %#"PRIx64"\n",
1569                      cpu, tid, sample->time, perf_time_to_tsc(sample->time,
1570                      &pt->tc));
1571
1572         ret = intel_pt_sync_switch(pt, cpu, tid, sample->time);
1573         if (ret <= 0)
1574                 return ret;
1575
1576         return machine__set_current_tid(pt->machine, cpu, -1, tid);
1577 }
1578
1579 static int intel_pt_context_switch(struct intel_pt *pt, union perf_event *event,
1580                                    struct perf_sample *sample)
1581 {
1582         bool out = event->header.misc & PERF_RECORD_MISC_SWITCH_OUT;
1583         pid_t pid, tid;
1584         int cpu, ret;
1585
1586         cpu = sample->cpu;
1587
1588         if (pt->have_sched_switch == 3) {
1589                 if (!out)
1590                         return 0;
1591                 if (event->header.type != PERF_RECORD_SWITCH_CPU_WIDE) {
1592                         pr_err("Expecting CPU-wide context switch event\n");
1593                         return -EINVAL;
1594                 }
1595                 pid = event->context_switch.next_prev_pid;
1596                 tid = event->context_switch.next_prev_tid;
1597         } else {
1598                 if (out)
1599                         return 0;
1600                 pid = sample->pid;
1601                 tid = sample->tid;
1602         }
1603
1604         if (tid == -1) {
1605                 pr_err("context_switch event has no tid\n");
1606                 return -EINVAL;
1607         }
1608
1609         intel_pt_log("context_switch: cpu %d pid %d tid %d time %"PRIu64" tsc %#"PRIx64"\n",
1610                      cpu, pid, tid, sample->time, perf_time_to_tsc(sample->time,
1611                      &pt->tc));
1612
1613         ret = intel_pt_sync_switch(pt, cpu, tid, sample->time);
1614         if (ret <= 0)
1615                 return ret;
1616
1617         return machine__set_current_tid(pt->machine, cpu, pid, tid);
1618 }
1619
1620 static int intel_pt_process_itrace_start(struct intel_pt *pt,
1621                                          union perf_event *event,
1622                                          struct perf_sample *sample)
1623 {
1624         if (!pt->per_cpu_mmaps)
1625                 return 0;
1626
1627         intel_pt_log("itrace_start: cpu %d pid %d tid %d time %"PRIu64" tsc %#"PRIx64"\n",
1628                      sample->cpu, event->itrace_start.pid,
1629                      event->itrace_start.tid, sample->time,
1630                      perf_time_to_tsc(sample->time, &pt->tc));
1631
1632         return machine__set_current_tid(pt->machine, sample->cpu,
1633                                         event->itrace_start.pid,
1634                                         event->itrace_start.tid);
1635 }
1636
1637 static int intel_pt_process_event(struct perf_session *session,
1638                                   union perf_event *event,
1639                                   struct perf_sample *sample,
1640                                   struct perf_tool *tool)
1641 {
1642         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
1643                                            auxtrace);
1644         u64 timestamp;
1645         int err = 0;
1646
1647         if (dump_trace)
1648                 return 0;
1649
1650         if (!tool->ordered_events) {
1651                 pr_err("Intel Processor Trace requires ordered events\n");
1652                 return -EINVAL;
1653         }
1654
1655         if (sample->time && sample->time != (u64)-1)
1656                 timestamp = perf_time_to_tsc(sample->time, &pt->tc);
1657         else
1658                 timestamp = 0;
1659
1660         if (timestamp || pt->timeless_decoding) {
1661                 err = intel_pt_update_queues(pt);
1662                 if (err)
1663                         return err;
1664         }
1665
1666         if (pt->timeless_decoding) {
1667                 if (event->header.type == PERF_RECORD_EXIT) {
1668                         err = intel_pt_process_timeless_queues(pt,
1669                                                                event->fork.tid,
1670                                                                sample->time);
1671                 }
1672         } else if (timestamp) {
1673                 err = intel_pt_process_queues(pt, timestamp);
1674         }
1675         if (err)
1676                 return err;
1677
1678         if (event->header.type == PERF_RECORD_AUX &&
1679             (event->aux.flags & PERF_AUX_FLAG_TRUNCATED) &&
1680             pt->synth_opts.errors) {
1681                 err = intel_pt_lost(pt, sample);
1682                 if (err)
1683                         return err;
1684         }
1685
1686         if (pt->switch_evsel && event->header.type == PERF_RECORD_SAMPLE)
1687                 err = intel_pt_process_switch(pt, sample);
1688         else if (event->header.type == PERF_RECORD_ITRACE_START)
1689                 err = intel_pt_process_itrace_start(pt, event, sample);
1690         else if (event->header.type == PERF_RECORD_SWITCH ||
1691                  event->header.type == PERF_RECORD_SWITCH_CPU_WIDE)
1692                 err = intel_pt_context_switch(pt, event, sample);
1693
1694         intel_pt_log("event %s (%u): cpu %d time %"PRIu64" tsc %#"PRIx64"\n",
1695                      perf_event__name(event->header.type), event->header.type,
1696                      sample->cpu, sample->time, timestamp);
1697
1698         return err;
1699 }
1700
1701 static int intel_pt_flush(struct perf_session *session, struct perf_tool *tool)
1702 {
1703         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
1704                                            auxtrace);
1705         int ret;
1706
1707         if (dump_trace)
1708                 return 0;
1709
1710         if (!tool->ordered_events)
1711                 return -EINVAL;
1712
1713         ret = intel_pt_update_queues(pt);
1714         if (ret < 0)
1715                 return ret;
1716
1717         if (pt->timeless_decoding)
1718                 return intel_pt_process_timeless_queues(pt, -1,
1719                                                         MAX_TIMESTAMP - 1);
1720
1721         return intel_pt_process_queues(pt, MAX_TIMESTAMP);
1722 }
1723
1724 static void intel_pt_free_events(struct perf_session *session)
1725 {
1726         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
1727                                            auxtrace);
1728         struct auxtrace_queues *queues = &pt->queues;
1729         unsigned int i;
1730
1731         for (i = 0; i < queues->nr_queues; i++) {
1732                 intel_pt_free_queue(queues->queue_array[i].priv);
1733                 queues->queue_array[i].priv = NULL;
1734         }
1735         intel_pt_log_disable();
1736         auxtrace_queues__free(queues);
1737 }
1738
1739 static void intel_pt_free(struct perf_session *session)
1740 {
1741         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
1742                                            auxtrace);
1743
1744         auxtrace_heap__free(&pt->heap);
1745         intel_pt_free_events(session);
1746         session->auxtrace = NULL;
1747         thread__delete(pt->unknown_thread);
1748         free(pt);
1749 }
1750
1751 static int intel_pt_process_auxtrace_event(struct perf_session *session,
1752                                            union perf_event *event,
1753                                            struct perf_tool *tool __maybe_unused)
1754 {
1755         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
1756                                            auxtrace);
1757
1758         if (pt->sampling_mode)
1759                 return 0;
1760
1761         if (!pt->data_queued) {
1762                 struct auxtrace_buffer *buffer;
1763                 off_t data_offset;
1764                 int fd = perf_data_file__fd(session->file);
1765                 int err;
1766
1767                 if (perf_data_file__is_pipe(session->file)) {
1768                         data_offset = 0;
1769                 } else {
1770                         data_offset = lseek(fd, 0, SEEK_CUR);
1771                         if (data_offset == -1)
1772                                 return -errno;
1773                 }
1774
1775                 err = auxtrace_queues__add_event(&pt->queues, session, event,
1776                                                  data_offset, &buffer);
1777                 if (err)
1778                         return err;
1779
1780                 /* Dump here now we have copied a piped trace out of the pipe */
1781                 if (dump_trace) {
1782                         if (auxtrace_buffer__get_data(buffer, fd)) {
1783                                 intel_pt_dump_event(pt, buffer->data,
1784                                                     buffer->size);
1785                                 auxtrace_buffer__put_data(buffer);
1786                         }
1787                 }
1788         }
1789
1790         return 0;
1791 }
1792
1793 struct intel_pt_synth {
1794         struct perf_tool dummy_tool;
1795         struct perf_session *session;
1796 };
1797
1798 static int intel_pt_event_synth(struct perf_tool *tool,
1799                                 union perf_event *event,
1800                                 struct perf_sample *sample __maybe_unused,
1801                                 struct machine *machine __maybe_unused)
1802 {
1803         struct intel_pt_synth *intel_pt_synth =
1804                         container_of(tool, struct intel_pt_synth, dummy_tool);
1805
1806         return perf_session__deliver_synth_event(intel_pt_synth->session, event,
1807                                                  NULL);
1808 }
1809
1810 static int intel_pt_synth_event(struct perf_session *session,
1811                                 struct perf_event_attr *attr, u64 id)
1812 {
1813         struct intel_pt_synth intel_pt_synth;
1814
1815         memset(&intel_pt_synth, 0, sizeof(struct intel_pt_synth));
1816         intel_pt_synth.session = session;
1817
1818         return perf_event__synthesize_attr(&intel_pt_synth.dummy_tool, attr, 1,
1819                                            &id, intel_pt_event_synth);
1820 }
1821
1822 static int intel_pt_synth_events(struct intel_pt *pt,
1823                                  struct perf_session *session)
1824 {
1825         struct perf_evlist *evlist = session->evlist;
1826         struct perf_evsel *evsel;
1827         struct perf_event_attr attr;
1828         bool found = false;
1829         u64 id;
1830         int err;
1831
1832         evlist__for_each(evlist, evsel) {
1833                 if (evsel->attr.type == pt->pmu_type && evsel->ids) {
1834                         found = true;
1835                         break;
1836                 }
1837         }
1838
1839         if (!found) {
1840                 pr_debug("There are no selected events with Intel Processor Trace data\n");
1841                 return 0;
1842         }
1843
1844         memset(&attr, 0, sizeof(struct perf_event_attr));
1845         attr.size = sizeof(struct perf_event_attr);
1846         attr.type = PERF_TYPE_HARDWARE;
1847         attr.sample_type = evsel->attr.sample_type & PERF_SAMPLE_MASK;
1848         attr.sample_type |= PERF_SAMPLE_IP | PERF_SAMPLE_TID |
1849                             PERF_SAMPLE_PERIOD;
1850         if (pt->timeless_decoding)
1851                 attr.sample_type &= ~(u64)PERF_SAMPLE_TIME;
1852         else
1853                 attr.sample_type |= PERF_SAMPLE_TIME;
1854         if (!pt->per_cpu_mmaps)
1855                 attr.sample_type &= ~(u64)PERF_SAMPLE_CPU;
1856         attr.exclude_user = evsel->attr.exclude_user;
1857         attr.exclude_kernel = evsel->attr.exclude_kernel;
1858         attr.exclude_hv = evsel->attr.exclude_hv;
1859         attr.exclude_host = evsel->attr.exclude_host;
1860         attr.exclude_guest = evsel->attr.exclude_guest;
1861         attr.sample_id_all = evsel->attr.sample_id_all;
1862         attr.read_format = evsel->attr.read_format;
1863
1864         id = evsel->id[0] + 1000000000;
1865         if (!id)
1866                 id = 1;
1867
1868         if (pt->synth_opts.instructions) {
1869                 attr.config = PERF_COUNT_HW_INSTRUCTIONS;
1870                 if (pt->synth_opts.period_type == PERF_ITRACE_PERIOD_NANOSECS)
1871                         attr.sample_period =
1872                                 intel_pt_ns_to_ticks(pt, pt->synth_opts.period);
1873                 else
1874                         attr.sample_period = pt->synth_opts.period;
1875                 pt->instructions_sample_period = attr.sample_period;
1876                 if (pt->synth_opts.callchain)
1877                         attr.sample_type |= PERF_SAMPLE_CALLCHAIN;
1878                 if (pt->synth_opts.last_branch)
1879                         attr.sample_type |= PERF_SAMPLE_BRANCH_STACK;
1880                 pr_debug("Synthesizing 'instructions' event with id %" PRIu64 " sample type %#" PRIx64 "\n",
1881                          id, (u64)attr.sample_type);
1882                 err = intel_pt_synth_event(session, &attr, id);
1883                 if (err) {
1884                         pr_err("%s: failed to synthesize 'instructions' event type\n",
1885                                __func__);
1886                         return err;
1887                 }
1888                 pt->sample_instructions = true;
1889                 pt->instructions_sample_type = attr.sample_type;
1890                 pt->instructions_id = id;
1891                 id += 1;
1892         }
1893
1894         if (pt->synth_opts.transactions) {
1895                 attr.config = PERF_COUNT_HW_INSTRUCTIONS;
1896                 attr.sample_period = 1;
1897                 if (pt->synth_opts.callchain)
1898                         attr.sample_type |= PERF_SAMPLE_CALLCHAIN;
1899                 if (pt->synth_opts.last_branch)
1900                         attr.sample_type |= PERF_SAMPLE_BRANCH_STACK;
1901                 pr_debug("Synthesizing 'transactions' event with id %" PRIu64 " sample type %#" PRIx64 "\n",
1902                          id, (u64)attr.sample_type);
1903                 err = intel_pt_synth_event(session, &attr, id);
1904                 if (err) {
1905                         pr_err("%s: failed to synthesize 'transactions' event type\n",
1906                                __func__);
1907                         return err;
1908                 }
1909                 pt->sample_transactions = true;
1910                 pt->transactions_id = id;
1911                 id += 1;
1912                 evlist__for_each(evlist, evsel) {
1913                         if (evsel->id && evsel->id[0] == pt->transactions_id) {
1914                                 if (evsel->name)
1915                                         zfree(&evsel->name);
1916                                 evsel->name = strdup("transactions");
1917                                 break;
1918                         }
1919                 }
1920         }
1921
1922         if (pt->synth_opts.branches) {
1923                 attr.config = PERF_COUNT_HW_BRANCH_INSTRUCTIONS;
1924                 attr.sample_period = 1;
1925                 attr.sample_type |= PERF_SAMPLE_ADDR;
1926                 attr.sample_type &= ~(u64)PERF_SAMPLE_CALLCHAIN;
1927                 attr.sample_type &= ~(u64)PERF_SAMPLE_BRANCH_STACK;
1928                 pr_debug("Synthesizing 'branches' event with id %" PRIu64 " sample type %#" PRIx64 "\n",
1929                          id, (u64)attr.sample_type);
1930                 err = intel_pt_synth_event(session, &attr, id);
1931                 if (err) {
1932                         pr_err("%s: failed to synthesize 'branches' event type\n",
1933                                __func__);
1934                         return err;
1935                 }
1936                 pt->sample_branches = true;
1937                 pt->branches_sample_type = attr.sample_type;
1938                 pt->branches_id = id;
1939         }
1940
1941         pt->synth_needs_swap = evsel->needs_swap;
1942
1943         return 0;
1944 }
1945
1946 static struct perf_evsel *intel_pt_find_sched_switch(struct perf_evlist *evlist)
1947 {
1948         struct perf_evsel *evsel;
1949
1950         evlist__for_each_reverse(evlist, evsel) {
1951                 const char *name = perf_evsel__name(evsel);
1952
1953                 if (!strcmp(name, "sched:sched_switch"))
1954                         return evsel;
1955         }
1956
1957         return NULL;
1958 }
1959
1960 static bool intel_pt_find_switch(struct perf_evlist *evlist)
1961 {
1962         struct perf_evsel *evsel;
1963
1964         evlist__for_each(evlist, evsel) {
1965                 if (evsel->attr.context_switch)
1966                         return true;
1967         }
1968
1969         return false;
1970 }
1971
1972 static int intel_pt_perf_config(const char *var, const char *value, void *data)
1973 {
1974         struct intel_pt *pt = data;
1975
1976         if (!strcmp(var, "intel-pt.mispred-all"))
1977                 pt->mispred_all = perf_config_bool(var, value);
1978
1979         return 0;
1980 }
1981
1982 static const char * const intel_pt_info_fmts[] = {
1983         [INTEL_PT_PMU_TYPE]             = "  PMU Type            %"PRId64"\n",
1984         [INTEL_PT_TIME_SHIFT]           = "  Time Shift          %"PRIu64"\n",
1985         [INTEL_PT_TIME_MULT]            = "  Time Muliplier      %"PRIu64"\n",
1986         [INTEL_PT_TIME_ZERO]            = "  Time Zero           %"PRIu64"\n",
1987         [INTEL_PT_CAP_USER_TIME_ZERO]   = "  Cap Time Zero       %"PRId64"\n",
1988         [INTEL_PT_TSC_BIT]              = "  TSC bit             %#"PRIx64"\n",
1989         [INTEL_PT_NORETCOMP_BIT]        = "  NoRETComp bit       %#"PRIx64"\n",
1990         [INTEL_PT_HAVE_SCHED_SWITCH]    = "  Have sched_switch   %"PRId64"\n",
1991         [INTEL_PT_SNAPSHOT_MODE]        = "  Snapshot mode       %"PRId64"\n",
1992         [INTEL_PT_PER_CPU_MMAPS]        = "  Per-cpu maps        %"PRId64"\n",
1993         [INTEL_PT_MTC_BIT]              = "  MTC bit             %#"PRIx64"\n",
1994         [INTEL_PT_TSC_CTC_N]            = "  TSC:CTC numerator   %"PRIu64"\n",
1995         [INTEL_PT_TSC_CTC_D]            = "  TSC:CTC denominator %"PRIu64"\n",
1996         [INTEL_PT_CYC_BIT]              = "  CYC bit             %#"PRIx64"\n",
1997 };
1998
1999 static void intel_pt_print_info(u64 *arr, int start, int finish)
2000 {
2001         int i;
2002
2003         if (!dump_trace)
2004                 return;
2005
2006         for (i = start; i <= finish; i++)
2007                 fprintf(stdout, intel_pt_info_fmts[i], arr[i]);
2008 }
2009
2010 int intel_pt_process_auxtrace_info(union perf_event *event,
2011                                    struct perf_session *session)
2012 {
2013         struct auxtrace_info_event *auxtrace_info = &event->auxtrace_info;
2014         size_t min_sz = sizeof(u64) * INTEL_PT_PER_CPU_MMAPS;
2015         struct intel_pt *pt;
2016         int err;
2017
2018         if (auxtrace_info->header.size < sizeof(struct auxtrace_info_event) +
2019                                         min_sz)
2020                 return -EINVAL;
2021
2022         pt = zalloc(sizeof(struct intel_pt));
2023         if (!pt)
2024                 return -ENOMEM;
2025
2026         perf_config(intel_pt_perf_config, pt);
2027
2028         err = auxtrace_queues__init(&pt->queues);
2029         if (err)
2030                 goto err_free;
2031
2032         intel_pt_log_set_name(INTEL_PT_PMU_NAME);
2033
2034         pt->session = session;
2035         pt->machine = &session->machines.host; /* No kvm support */
2036         pt->auxtrace_type = auxtrace_info->type;
2037         pt->pmu_type = auxtrace_info->priv[INTEL_PT_PMU_TYPE];
2038         pt->tc.time_shift = auxtrace_info->priv[INTEL_PT_TIME_SHIFT];
2039         pt->tc.time_mult = auxtrace_info->priv[INTEL_PT_TIME_MULT];
2040         pt->tc.time_zero = auxtrace_info->priv[INTEL_PT_TIME_ZERO];
2041         pt->cap_user_time_zero = auxtrace_info->priv[INTEL_PT_CAP_USER_TIME_ZERO];
2042         pt->tsc_bit = auxtrace_info->priv[INTEL_PT_TSC_BIT];
2043         pt->noretcomp_bit = auxtrace_info->priv[INTEL_PT_NORETCOMP_BIT];
2044         pt->have_sched_switch = auxtrace_info->priv[INTEL_PT_HAVE_SCHED_SWITCH];
2045         pt->snapshot_mode = auxtrace_info->priv[INTEL_PT_SNAPSHOT_MODE];
2046         pt->per_cpu_mmaps = auxtrace_info->priv[INTEL_PT_PER_CPU_MMAPS];
2047         intel_pt_print_info(&auxtrace_info->priv[0], INTEL_PT_PMU_TYPE,
2048                             INTEL_PT_PER_CPU_MMAPS);
2049
2050         if (auxtrace_info->header.size >= sizeof(struct auxtrace_info_event) +
2051                                         (sizeof(u64) * INTEL_PT_CYC_BIT)) {
2052                 pt->mtc_bit = auxtrace_info->priv[INTEL_PT_MTC_BIT];
2053                 pt->mtc_freq_bits = auxtrace_info->priv[INTEL_PT_MTC_FREQ_BITS];
2054                 pt->tsc_ctc_ratio_n = auxtrace_info->priv[INTEL_PT_TSC_CTC_N];
2055                 pt->tsc_ctc_ratio_d = auxtrace_info->priv[INTEL_PT_TSC_CTC_D];
2056                 pt->cyc_bit = auxtrace_info->priv[INTEL_PT_CYC_BIT];
2057                 intel_pt_print_info(&auxtrace_info->priv[0], INTEL_PT_MTC_BIT,
2058                                     INTEL_PT_CYC_BIT);
2059         }
2060
2061         pt->timeless_decoding = intel_pt_timeless_decoding(pt);
2062         pt->have_tsc = intel_pt_have_tsc(pt);
2063         pt->sampling_mode = false;
2064         pt->est_tsc = !pt->timeless_decoding;
2065
2066         pt->unknown_thread = thread__new(999999999, 999999999);
2067         if (!pt->unknown_thread) {
2068                 err = -ENOMEM;
2069                 goto err_free_queues;
2070         }
2071         err = thread__set_comm(pt->unknown_thread, "unknown", 0);
2072         if (err)
2073                 goto err_delete_thread;
2074         if (thread__init_map_groups(pt->unknown_thread, pt->machine)) {
2075                 err = -ENOMEM;
2076                 goto err_delete_thread;
2077         }
2078
2079         pt->auxtrace.process_event = intel_pt_process_event;
2080         pt->auxtrace.process_auxtrace_event = intel_pt_process_auxtrace_event;
2081         pt->auxtrace.flush_events = intel_pt_flush;
2082         pt->auxtrace.free_events = intel_pt_free_events;
2083         pt->auxtrace.free = intel_pt_free;
2084         session->auxtrace = &pt->auxtrace;
2085
2086         if (dump_trace)
2087                 return 0;
2088
2089         if (pt->have_sched_switch == 1) {
2090                 pt->switch_evsel = intel_pt_find_sched_switch(session->evlist);
2091                 if (!pt->switch_evsel) {
2092                         pr_err("%s: missing sched_switch event\n", __func__);
2093                         goto err_delete_thread;
2094                 }
2095         } else if (pt->have_sched_switch == 2 &&
2096                    !intel_pt_find_switch(session->evlist)) {
2097                 pr_err("%s: missing context_switch attribute flag\n", __func__);
2098                 goto err_delete_thread;
2099         }
2100
2101         if (session->itrace_synth_opts && session->itrace_synth_opts->set) {
2102                 pt->synth_opts = *session->itrace_synth_opts;
2103         } else {
2104                 itrace_synth_opts__set_default(&pt->synth_opts);
2105                 if (use_browser != -1) {
2106                         pt->synth_opts.branches = false;
2107                         pt->synth_opts.callchain = true;
2108                 }
2109         }
2110
2111         if (pt->synth_opts.log)
2112                 intel_pt_log_enable();
2113
2114         /* Maximum non-turbo ratio is TSC freq / 100 MHz */
2115         if (pt->tc.time_mult) {
2116                 u64 tsc_freq = intel_pt_ns_to_ticks(pt, 1000000000);
2117
2118                 pt->max_non_turbo_ratio = (tsc_freq + 50000000) / 100000000;
2119                 intel_pt_log("TSC frequency %"PRIu64"\n", tsc_freq);
2120                 intel_pt_log("Maximum non-turbo ratio %u\n",
2121                              pt->max_non_turbo_ratio);
2122         }
2123
2124         if (pt->synth_opts.calls)
2125                 pt->branches_filter |= PERF_IP_FLAG_CALL | PERF_IP_FLAG_ASYNC |
2126                                        PERF_IP_FLAG_TRACE_END;
2127         if (pt->synth_opts.returns)
2128                 pt->branches_filter |= PERF_IP_FLAG_RETURN |
2129                                        PERF_IP_FLAG_TRACE_BEGIN;
2130
2131         if (pt->synth_opts.callchain && !symbol_conf.use_callchain) {
2132                 symbol_conf.use_callchain = true;
2133                 if (callchain_register_param(&callchain_param) < 0) {
2134                         symbol_conf.use_callchain = false;
2135                         pt->synth_opts.callchain = false;
2136                 }
2137         }
2138
2139         err = intel_pt_synth_events(pt, session);
2140         if (err)
2141                 goto err_delete_thread;
2142
2143         err = auxtrace_queues__process_index(&pt->queues, session);
2144         if (err)
2145                 goto err_delete_thread;
2146
2147         if (pt->queues.populated)
2148                 pt->data_queued = true;
2149
2150         if (pt->timeless_decoding)
2151                 pr_debug2("Intel PT decoding without timestamps\n");
2152
2153         return 0;
2154
2155 err_delete_thread:
2156         thread__delete(pt->unknown_thread);
2157 err_free_queues:
2158         intel_pt_log_disable();
2159         auxtrace_queues__free(&pt->queues);
2160         session->auxtrace = NULL;
2161 err_free:
2162         free(pt);
2163         return err;
2164 }