| /* |
| * Copyright 2014, Michael Ellerman, IBM Corp. |
| * Licensed under GPLv2. |
| */ |
| |
| #define _GNU_SOURCE /* For CPU_ZERO etc. */ |
| |
| #include <sched.h> |
| #include <sys/wait.h> |
| #include <setjmp.h> |
| #include <signal.h> |
| #include <stdio.h> |
| #include <stdlib.h> |
| #include <string.h> |
| #include <sys/ioctl.h> |
| |
| #include "trace.h" |
| #include "reg.h" |
| #include "ebb.h" |
| |
| |
| void (*ebb_user_func)(void); |
| |
| void ebb_hook(void) |
| { |
| if (ebb_user_func) |
| ebb_user_func(); |
| } |
| |
| struct ebb_state ebb_state; |
| |
| u64 sample_period = 0x40000000ull; |
| |
| void reset_ebb_with_clear_mask(unsigned long mmcr0_clear_mask) |
| { |
| u64 val; |
| |
| /* 2) clear MMCR0[PMAO] - docs say BESCR[PMEO] should do this */ |
| /* 3) set MMCR0[PMAE] - docs say BESCR[PME] should do this */ |
| val = mfspr(SPRN_MMCR0); |
| mtspr(SPRN_MMCR0, (val & ~mmcr0_clear_mask) | MMCR0_PMAE); |
| |
| /* 4) clear BESCR[PMEO] */ |
| mtspr(SPRN_BESCRR, BESCR_PMEO); |
| |
| /* 5) set BESCR[PME] */ |
| mtspr(SPRN_BESCRS, BESCR_PME); |
| |
| /* 6) rfebb 1 - done in our caller */ |
| } |
| |
| void reset_ebb(void) |
| { |
| reset_ebb_with_clear_mask(MMCR0_PMAO | MMCR0_FC); |
| } |
| |
| /* Called outside of the EBB handler to check MMCR0 is sane */ |
| int ebb_check_mmcr0(void) |
| { |
| u64 val; |
| |
| val = mfspr(SPRN_MMCR0); |
| if ((val & (MMCR0_FC | MMCR0_PMAO)) == MMCR0_FC) { |
| /* It's OK if we see FC & PMAO, but not FC by itself */ |
| printf("Outside of loop, only FC set 0x%llx\n", val); |
| return 1; |
| } |
| |
| return 0; |
| } |
| |
| bool ebb_check_count(int pmc, u64 sample_period, int fudge) |
| { |
| u64 count, upper, lower; |
| |
| count = ebb_state.stats.pmc_count[PMC_INDEX(pmc)]; |
| |
| lower = ebb_state.stats.ebb_count * (sample_period - fudge); |
| |
| if (count < lower) { |
| printf("PMC%d count (0x%llx) below lower limit 0x%llx (-0x%llx)\n", |
| pmc, count, lower, lower - count); |
| return false; |
| } |
| |
| upper = ebb_state.stats.ebb_count * (sample_period + fudge); |
| |
| if (count > upper) { |
| printf("PMC%d count (0x%llx) above upper limit 0x%llx (+0x%llx)\n", |
| pmc, count, upper, count - upper); |
| return false; |
| } |
| |
| printf("PMC%d count (0x%llx) is between 0x%llx and 0x%llx delta +0x%llx/-0x%llx\n", |
| pmc, count, lower, upper, count - lower, upper - count); |
| |
| return true; |
| } |
| |
| void standard_ebb_callee(void) |
| { |
| int found, i; |
| u64 val; |
| |
| val = mfspr(SPRN_BESCR); |
| if (!(val & BESCR_PMEO)) { |
| ebb_state.stats.spurious++; |
| goto out; |
| } |
| |
| ebb_state.stats.ebb_count++; |
| trace_log_counter(ebb_state.trace, ebb_state.stats.ebb_count); |
| |
| val = mfspr(SPRN_MMCR0); |
| trace_log_reg(ebb_state.trace, SPRN_MMCR0, val); |
| |
| found = 0; |
| for (i = 1; i <= 6; i++) { |
| if (ebb_state.pmc_enable[PMC_INDEX(i)]) |
| found += count_pmc(i, sample_period); |
| } |
| |
| if (!found) |
| ebb_state.stats.no_overflow++; |
| |
| out: |
| reset_ebb(); |
| } |
| |
| extern void ebb_handler(void); |
| |
| void setup_ebb_handler(void (*callee)(void)) |
| { |
| u64 entry; |
| |
| #if defined(_CALL_ELF) && _CALL_ELF == 2 |
| entry = (u64)ebb_handler; |
| #else |
| struct opd |
| { |
| u64 entry; |
| u64 toc; |
| } *opd; |
| |
| opd = (struct opd *)ebb_handler; |
| entry = opd->entry; |
| #endif |
| printf("EBB Handler is at %#llx\n", entry); |
| |
| ebb_user_func = callee; |
| |
| /* Ensure ebb_user_func is set before we set the handler */ |
| mb(); |
| mtspr(SPRN_EBBHR, entry); |
| |
| /* Make sure the handler is set before we return */ |
| mb(); |
| } |
| |
| void clear_ebb_stats(void) |
| { |
| memset(&ebb_state.stats, 0, sizeof(ebb_state.stats)); |
| } |
| |
| void dump_summary_ebb_state(void) |
| { |
| printf("ebb_state:\n" \ |
| " ebb_count = %d\n" \ |
| " spurious = %d\n" \ |
| " negative = %d\n" \ |
| " no_overflow = %d\n" \ |
| " pmc[1] count = 0x%llx\n" \ |
| " pmc[2] count = 0x%llx\n" \ |
| " pmc[3] count = 0x%llx\n" \ |
| " pmc[4] count = 0x%llx\n" \ |
| " pmc[5] count = 0x%llx\n" \ |
| " pmc[6] count = 0x%llx\n", |
| ebb_state.stats.ebb_count, ebb_state.stats.spurious, |
| ebb_state.stats.negative, ebb_state.stats.no_overflow, |
| ebb_state.stats.pmc_count[0], ebb_state.stats.pmc_count[1], |
| ebb_state.stats.pmc_count[2], ebb_state.stats.pmc_count[3], |
| ebb_state.stats.pmc_count[4], ebb_state.stats.pmc_count[5]); |
| } |
| |
| static char *decode_mmcr0(u32 value) |
| { |
| static char buf[16]; |
| |
| buf[0] = '\0'; |
| |
| if (value & (1 << 31)) |
| strcat(buf, "FC "); |
| if (value & (1 << 26)) |
| strcat(buf, "PMAE "); |
| if (value & (1 << 7)) |
| strcat(buf, "PMAO "); |
| |
| return buf; |
| } |
| |
| static char *decode_bescr(u64 value) |
| { |
| static char buf[16]; |
| |
| buf[0] = '\0'; |
| |
| if (value & (1ull << 63)) |
| strcat(buf, "GE "); |
| if (value & (1ull << 32)) |
| strcat(buf, "PMAE "); |
| if (value & 1) |
| strcat(buf, "PMAO "); |
| |
| return buf; |
| } |
| |
| void dump_ebb_hw_state(void) |
| { |
| u64 bescr; |
| u32 mmcr0; |
| |
| mmcr0 = mfspr(SPRN_MMCR0); |
| bescr = mfspr(SPRN_BESCR); |
| |
| printf("HW state:\n" \ |
| "MMCR0 0x%016x %s\n" \ |
| "EBBHR 0x%016lx\n" \ |
| "BESCR 0x%016llx %s\n" \ |
| "PMC1 0x%016lx\n" \ |
| "PMC2 0x%016lx\n" \ |
| "PMC3 0x%016lx\n" \ |
| "PMC4 0x%016lx\n" \ |
| "PMC5 0x%016lx\n" \ |
| "PMC6 0x%016lx\n" \ |
| "SIAR 0x%016lx\n", |
| mmcr0, decode_mmcr0(mmcr0), mfspr(SPRN_EBBHR), bescr, |
| decode_bescr(bescr), mfspr(SPRN_PMC1), mfspr(SPRN_PMC2), |
| mfspr(SPRN_PMC3), mfspr(SPRN_PMC4), mfspr(SPRN_PMC5), |
| mfspr(SPRN_PMC6), mfspr(SPRN_SIAR)); |
| } |
| |
| void dump_ebb_state(void) |
| { |
| dump_summary_ebb_state(); |
| |
| dump_ebb_hw_state(); |
| |
| trace_buffer_print(ebb_state.trace); |
| } |
| |
| int count_pmc(int pmc, uint32_t sample_period) |
| { |
| uint32_t start_value; |
| u64 val; |
| |
| /* 0) Read PMC */ |
| start_value = pmc_sample_period(sample_period); |
| |
| val = read_pmc(pmc); |
| if (val < start_value) |
| ebb_state.stats.negative++; |
| else |
| ebb_state.stats.pmc_count[PMC_INDEX(pmc)] += val - start_value; |
| |
| trace_log_reg(ebb_state.trace, SPRN_PMC1 + pmc - 1, val); |
| |
| /* 1) Reset PMC */ |
| write_pmc(pmc, start_value); |
| |
| /* Report if we overflowed */ |
| return val >= COUNTER_OVERFLOW; |
| } |
| |
| int ebb_event_enable(struct event *e) |
| { |
| int rc; |
| |
| /* Ensure any SPR writes are ordered vs us */ |
| mb(); |
| |
| rc = ioctl(e->fd, PERF_EVENT_IOC_ENABLE); |
| if (rc) |
| return rc; |
| |
| rc = event_read(e); |
| |
| /* Ditto */ |
| mb(); |
| |
| return rc; |
| } |
| |
| void ebb_freeze_pmcs(void) |
| { |
| mtspr(SPRN_MMCR0, mfspr(SPRN_MMCR0) | MMCR0_FC); |
| mb(); |
| } |
| |
| void ebb_unfreeze_pmcs(void) |
| { |
| /* Unfreeze counters */ |
| mtspr(SPRN_MMCR0, mfspr(SPRN_MMCR0) & ~MMCR0_FC); |
| mb(); |
| } |
| |
| void ebb_global_enable(void) |
| { |
| /* Enable EBBs globally and PMU EBBs */ |
| mtspr(SPRN_BESCR, 0x8000000100000000ull); |
| mb(); |
| } |
| |
| void ebb_global_disable(void) |
| { |
| /* Disable EBBs & freeze counters, events are still scheduled */ |
| mtspr(SPRN_BESCRR, BESCR_PME); |
| mb(); |
| } |
| |
| void event_ebb_init(struct event *e) |
| { |
| e->attr.config |= (1ull << 63); |
| } |
| |
| void event_bhrb_init(struct event *e, unsigned ifm) |
| { |
| e->attr.config |= (1ull << 62) | ((u64)ifm << 60); |
| } |
| |
| void event_leader_ebb_init(struct event *e) |
| { |
| event_ebb_init(e); |
| |
| e->attr.exclusive = 1; |
| e->attr.pinned = 1; |
| } |
| |
| int core_busy_loop(void) |
| { |
| int rc; |
| |
| asm volatile ( |
| "li 3, 0x3030\n" |
| "std 3, -96(1)\n" |
| "li 4, 0x4040\n" |
| "std 4, -104(1)\n" |
| "li 5, 0x5050\n" |
| "std 5, -112(1)\n" |
| "li 6, 0x6060\n" |
| "std 6, -120(1)\n" |
| "li 7, 0x7070\n" |
| "std 7, -128(1)\n" |
| "li 8, 0x0808\n" |
| "std 8, -136(1)\n" |
| "li 9, 0x0909\n" |
| "std 9, -144(1)\n" |
| "li 10, 0x1010\n" |
| "std 10, -152(1)\n" |
| "li 11, 0x1111\n" |
| "std 11, -160(1)\n" |
| "li 14, 0x1414\n" |
| "std 14, -168(1)\n" |
| "li 15, 0x1515\n" |
| "std 15, -176(1)\n" |
| "li 16, 0x1616\n" |
| "std 16, -184(1)\n" |
| "li 17, 0x1717\n" |
| "std 17, -192(1)\n" |
| "li 18, 0x1818\n" |
| "std 18, -200(1)\n" |
| "li 19, 0x1919\n" |
| "std 19, -208(1)\n" |
| "li 20, 0x2020\n" |
| "std 20, -216(1)\n" |
| "li 21, 0x2121\n" |
| "std 21, -224(1)\n" |
| "li 22, 0x2222\n" |
| "std 22, -232(1)\n" |
| "li 23, 0x2323\n" |
| "std 23, -240(1)\n" |
| "li 24, 0x2424\n" |
| "std 24, -248(1)\n" |
| "li 25, 0x2525\n" |
| "std 25, -256(1)\n" |
| "li 26, 0x2626\n" |
| "std 26, -264(1)\n" |
| "li 27, 0x2727\n" |
| "std 27, -272(1)\n" |
| "li 28, 0x2828\n" |
| "std 28, -280(1)\n" |
| "li 29, 0x2929\n" |
| "std 29, -288(1)\n" |
| "li 30, 0x3030\n" |
| "li 31, 0x3131\n" |
| |
| "li 3, 0\n" |
| "0: " |
| "addi 3, 3, 1\n" |
| "cmpwi 3, 100\n" |
| "blt 0b\n" |
| |
| /* Return 1 (fail) unless we get through all the checks */ |
| "li 0, 1\n" |
| |
| /* Check none of our registers have been corrupted */ |
| "cmpwi 4, 0x4040\n" |
| "bne 1f\n" |
| "cmpwi 5, 0x5050\n" |
| "bne 1f\n" |
| "cmpwi 6, 0x6060\n" |
| "bne 1f\n" |
| "cmpwi 7, 0x7070\n" |
| "bne 1f\n" |
| "cmpwi 8, 0x0808\n" |
| "bne 1f\n" |
| "cmpwi 9, 0x0909\n" |
| "bne 1f\n" |
| "cmpwi 10, 0x1010\n" |
| "bne 1f\n" |
| "cmpwi 11, 0x1111\n" |
| "bne 1f\n" |
| "cmpwi 14, 0x1414\n" |
| "bne 1f\n" |
| "cmpwi 15, 0x1515\n" |
| "bne 1f\n" |
| "cmpwi 16, 0x1616\n" |
| "bne 1f\n" |
| "cmpwi 17, 0x1717\n" |
| "bne 1f\n" |
| "cmpwi 18, 0x1818\n" |
| "bne 1f\n" |
| "cmpwi 19, 0x1919\n" |
| "bne 1f\n" |
| "cmpwi 20, 0x2020\n" |
| "bne 1f\n" |
| "cmpwi 21, 0x2121\n" |
| "bne 1f\n" |
| "cmpwi 22, 0x2222\n" |
| "bne 1f\n" |
| "cmpwi 23, 0x2323\n" |
| "bne 1f\n" |
| "cmpwi 24, 0x2424\n" |
| "bne 1f\n" |
| "cmpwi 25, 0x2525\n" |
| "bne 1f\n" |
| "cmpwi 26, 0x2626\n" |
| "bne 1f\n" |
| "cmpwi 27, 0x2727\n" |
| "bne 1f\n" |
| "cmpwi 28, 0x2828\n" |
| "bne 1f\n" |
| "cmpwi 29, 0x2929\n" |
| "bne 1f\n" |
| "cmpwi 30, 0x3030\n" |
| "bne 1f\n" |
| "cmpwi 31, 0x3131\n" |
| "bne 1f\n" |
| |
| /* Load junk into all our registers before we reload them from the stack. */ |
| "li 3, 0xde\n" |
| "li 4, 0xad\n" |
| "li 5, 0xbe\n" |
| "li 6, 0xef\n" |
| "li 7, 0xde\n" |
| "li 8, 0xad\n" |
| "li 9, 0xbe\n" |
| "li 10, 0xef\n" |
| "li 11, 0xde\n" |
| "li 14, 0xad\n" |
| "li 15, 0xbe\n" |
| "li 16, 0xef\n" |
| "li 17, 0xde\n" |
| "li 18, 0xad\n" |
| "li 19, 0xbe\n" |
| "li 20, 0xef\n" |
| "li 21, 0xde\n" |
| "li 22, 0xad\n" |
| "li 23, 0xbe\n" |
| "li 24, 0xef\n" |
| "li 25, 0xde\n" |
| "li 26, 0xad\n" |
| "li 27, 0xbe\n" |
| "li 28, 0xef\n" |
| "li 29, 0xdd\n" |
| |
| "ld 3, -96(1)\n" |
| "cmpwi 3, 0x3030\n" |
| "bne 1f\n" |
| "ld 4, -104(1)\n" |
| "cmpwi 4, 0x4040\n" |
| "bne 1f\n" |
| "ld 5, -112(1)\n" |
| "cmpwi 5, 0x5050\n" |
| "bne 1f\n" |
| "ld 6, -120(1)\n" |
| "cmpwi 6, 0x6060\n" |
| "bne 1f\n" |
| "ld 7, -128(1)\n" |
| "cmpwi 7, 0x7070\n" |
| "bne 1f\n" |
| "ld 8, -136(1)\n" |
| "cmpwi 8, 0x0808\n" |
| "bne 1f\n" |
| "ld 9, -144(1)\n" |
| "cmpwi 9, 0x0909\n" |
| "bne 1f\n" |
| "ld 10, -152(1)\n" |
| "cmpwi 10, 0x1010\n" |
| "bne 1f\n" |
| "ld 11, -160(1)\n" |
| "cmpwi 11, 0x1111\n" |
| "bne 1f\n" |
| "ld 14, -168(1)\n" |
| "cmpwi 14, 0x1414\n" |
| "bne 1f\n" |
| "ld 15, -176(1)\n" |
| "cmpwi 15, 0x1515\n" |
| "bne 1f\n" |
| "ld 16, -184(1)\n" |
| "cmpwi 16, 0x1616\n" |
| "bne 1f\n" |
| "ld 17, -192(1)\n" |
| "cmpwi 17, 0x1717\n" |
| "bne 1f\n" |
| "ld 18, -200(1)\n" |
| "cmpwi 18, 0x1818\n" |
| "bne 1f\n" |
| "ld 19, -208(1)\n" |
| "cmpwi 19, 0x1919\n" |
| "bne 1f\n" |
| "ld 20, -216(1)\n" |
| "cmpwi 20, 0x2020\n" |
| "bne 1f\n" |
| "ld 21, -224(1)\n" |
| "cmpwi 21, 0x2121\n" |
| "bne 1f\n" |
| "ld 22, -232(1)\n" |
| "cmpwi 22, 0x2222\n" |
| "bne 1f\n" |
| "ld 23, -240(1)\n" |
| "cmpwi 23, 0x2323\n" |
| "bne 1f\n" |
| "ld 24, -248(1)\n" |
| "cmpwi 24, 0x2424\n" |
| "bne 1f\n" |
| "ld 25, -256(1)\n" |
| "cmpwi 25, 0x2525\n" |
| "bne 1f\n" |
| "ld 26, -264(1)\n" |
| "cmpwi 26, 0x2626\n" |
| "bne 1f\n" |
| "ld 27, -272(1)\n" |
| "cmpwi 27, 0x2727\n" |
| "bne 1f\n" |
| "ld 28, -280(1)\n" |
| "cmpwi 28, 0x2828\n" |
| "bne 1f\n" |
| "ld 29, -288(1)\n" |
| "cmpwi 29, 0x2929\n" |
| "bne 1f\n" |
| |
| /* Load 0 (success) to return */ |
| "li 0, 0\n" |
| |
| "1: mr %0, 0\n" |
| |
| : "=r" (rc) |
| : /* no inputs */ |
| : "3", "4", "5", "6", "7", "8", "9", "10", "11", "14", |
| "15", "16", "17", "18", "19", "20", "21", "22", "23", |
| "24", "25", "26", "27", "28", "29", "30", "31", |
| "memory" |
| ); |
| |
| return rc; |
| } |
| |
| int core_busy_loop_with_freeze(void) |
| { |
| int rc; |
| |
| mtspr(SPRN_MMCR0, mfspr(SPRN_MMCR0) & ~MMCR0_FC); |
| rc = core_busy_loop(); |
| mtspr(SPRN_MMCR0, mfspr(SPRN_MMCR0) | MMCR0_FC); |
| |
| return rc; |
| } |
| |
| int ebb_child(union pipe read_pipe, union pipe write_pipe) |
| { |
| struct event event; |
| uint64_t val; |
| |
| FAIL_IF(wait_for_parent(read_pipe)); |
| |
| event_init_named(&event, 0x1001e, "cycles"); |
| event_leader_ebb_init(&event); |
| |
| event.attr.exclude_kernel = 1; |
| event.attr.exclude_hv = 1; |
| event.attr.exclude_idle = 1; |
| |
| FAIL_IF(event_open(&event)); |
| |
| ebb_enable_pmc_counting(1); |
| setup_ebb_handler(standard_ebb_callee); |
| ebb_global_enable(); |
| |
| FAIL_IF(event_enable(&event)); |
| |
| if (event_read(&event)) { |
| /* |
| * Some tests expect to fail here, so don't report an error on |
| * this line, and return a distinguisable error code. Tell the |
| * parent an error happened. |
| */ |
| notify_parent_of_error(write_pipe); |
| return 2; |
| } |
| |
| mtspr(SPRN_PMC1, pmc_sample_period(sample_period)); |
| |
| FAIL_IF(notify_parent(write_pipe)); |
| FAIL_IF(wait_for_parent(read_pipe)); |
| FAIL_IF(notify_parent(write_pipe)); |
| |
| while (ebb_state.stats.ebb_count < 20) { |
| FAIL_IF(core_busy_loop()); |
| |
| /* To try and hit SIGILL case */ |
| val = mfspr(SPRN_MMCRA); |
| val |= mfspr(SPRN_MMCR2); |
| val |= mfspr(SPRN_MMCR0); |
| } |
| |
| ebb_global_disable(); |
| ebb_freeze_pmcs(); |
| |
| count_pmc(1, sample_period); |
| |
| dump_ebb_state(); |
| |
| event_close(&event); |
| |
| FAIL_IF(ebb_state.stats.ebb_count == 0); |
| |
| return 0; |
| } |
| |
| static jmp_buf setjmp_env; |
| |
| static void sigill_handler(int signal) |
| { |
| printf("Took sigill\n"); |
| longjmp(setjmp_env, 1); |
| } |
| |
| static struct sigaction sigill_action = { |
| .sa_handler = sigill_handler, |
| }; |
| |
| int catch_sigill(void (*func)(void)) |
| { |
| if (sigaction(SIGILL, &sigill_action, NULL)) { |
| perror("sigaction"); |
| return 1; |
| } |
| |
| if (setjmp(setjmp_env) == 0) { |
| func(); |
| return 1; |
| } |
| |
| return 0; |
| } |
| |
| void write_pmc1(void) |
| { |
| mtspr(SPRN_PMC1, 0); |
| } |
| |
| void write_pmc(int pmc, u64 value) |
| { |
| switch (pmc) { |
| case 1: mtspr(SPRN_PMC1, value); break; |
| case 2: mtspr(SPRN_PMC2, value); break; |
| case 3: mtspr(SPRN_PMC3, value); break; |
| case 4: mtspr(SPRN_PMC4, value); break; |
| case 5: mtspr(SPRN_PMC5, value); break; |
| case 6: mtspr(SPRN_PMC6, value); break; |
| } |
| } |
| |
| u64 read_pmc(int pmc) |
| { |
| switch (pmc) { |
| case 1: return mfspr(SPRN_PMC1); |
| case 2: return mfspr(SPRN_PMC2); |
| case 3: return mfspr(SPRN_PMC3); |
| case 4: return mfspr(SPRN_PMC4); |
| case 5: return mfspr(SPRN_PMC5); |
| case 6: return mfspr(SPRN_PMC6); |
| } |
| |
| return 0; |
| } |
| |
| static void term_handler(int signal) |
| { |
| dump_summary_ebb_state(); |
| dump_ebb_hw_state(); |
| abort(); |
| } |
| |
| struct sigaction term_action = { |
| .sa_handler = term_handler, |
| }; |
| |
| static void __attribute__((constructor)) ebb_init(void) |
| { |
| clear_ebb_stats(); |
| |
| if (sigaction(SIGTERM, &term_action, NULL)) |
| perror("sigaction"); |
| |
| ebb_state.trace = trace_buffer_allocate(1 * 1024 * 1024); |
| } |
