drivers/gator/gator_events_perf_pmu.c - arm/linux-arm-legacy - Git at Google

 /**
  * Copyright (C) ARM Limited 2010-2014. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
  * published by the Free Software Foundation.
  */

 #include "gator.h"

 // gator_events_armvX.c is used for Linux 2.6.x
 #if GATOR_PERF_PMU_SUPPORT

 #include <linux/io.h>
 #ifdef CONFIG_OF
 #include <linux/of_address.h>
 #endif
 #include <linux/perf_event.h>
 #include <linux/slab.h>

 extern bool event_based_sampling;

 // Maximum number of per-core counters - currently reserves enough space for two full hardware PMUs for big.LITTLE
 #define CNTMAX 16
 #define CCI_400 4
 // Maximum number of uncore counters
 // + 1 for the cci-400 cycles counter
 #define UCCNT (CCI_400 + 1)

 // Default to 0 if unable to probe the revision which was the previous behavior
 #define DEFAULT_CCI_REVISION 0

 // A gator_attr is needed for every counter
 struct gator_attr {
 	// Set once in gator_events_perf_pmu_*_init - the name of the event in the gatorfs
 	char name[40];
 	// Exposed in gatorfs - set by gatord to enable this counter
 	unsigned long enabled;
 	// Set once in gator_events_perf_pmu_*_init - the perf type to use, see perf_type_id in the perf_event.h header file.
 	unsigned long type;
 	// Exposed in gatorfs - set by gatord to select the event to collect
 	unsigned long event;
 	// Exposed in gatorfs - set by gatord with the sample period to use and enable EBS for this counter
 	unsigned long count;
 	// Exposed as read only in gatorfs - set once in __attr_init as the key to use in the APC data
 	unsigned long key;
 };

 // Per-core counter attributes
 static struct gator_attr attrs[CNTMAX];
 // Number of initialized per-core counters
 static int attr_count;
 // Uncore counter attributes
 static struct gator_attr uc_attrs[UCCNT];
 // Number of initialized uncore counters
 static int uc_attr_count;

 struct gator_event {
 	int curr;
 	int prev;
 	int prev_delta;
 	bool zero;
 	struct perf_event *pevent;
 	struct perf_event_attr *pevent_attr;
 };

 static DEFINE_PER_CPU(struct gator_event[CNTMAX], events);
 static struct gator_event uc_events[UCCNT];
 static DEFINE_PER_CPU(int[(CNTMAX + UCCNT)*2], perf_cnt);

 static void gator_events_perf_pmu_stop(void);

 static int __create_files(struct super_block *sb, struct dentry *root, struct gator_attr *const attr)
 {
 	struct dentry *dir;

 	if (attr->name[0] == '\0') {
 		return 0;
 	}
 	dir = gatorfs_mkdir(sb, root, attr->name);
 	if (!dir) {
 		return -1;
 	}
 	gatorfs_create_ulong(sb, dir, "enabled", &attr->enabled);
 	gatorfs_create_ulong(sb, dir, "count", &attr->count);
 	gatorfs_create_ro_ulong(sb, dir, "key", &attr->key);
 	gatorfs_create_ulong(sb, dir, "event", &attr->event);

 	return 0;
 }

 static int gator_events_perf_pmu_create_files(struct super_block *sb, struct dentry *root)
 {
 	int cnt;

 	for (cnt = 0; cnt < attr_count; cnt++) {
 		if (__create_files(sb, root, &attrs[cnt]) != 0) {
 			return -1;
 		}
 	}

 	for (cnt = 0; cnt < uc_attr_count; cnt++) {
 		if (__create_files(sb, root, &uc_attrs[cnt]) != 0) {
 			return -1;
 		}
 	}

 	return 0;
 }

 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 1, 0)
 static void ebs_overflow_handler(struct perf_event *event, int unused, struct perf_sample_data *data, struct pt_regs *regs)
 #else
 static void ebs_overflow_handler(struct perf_event *event, struct perf_sample_data *data, struct pt_regs *regs)
 #endif
 {
 	gator_backtrace_handler(regs);
 }

 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 1, 0)
 static void dummy_handler(struct perf_event *event, int unused, struct perf_sample_data *data, struct pt_regs *regs)
 #else
 static void dummy_handler(struct perf_event *event, struct perf_sample_data *data, struct pt_regs *regs)
 #endif
 {
 // Required as perf_event_create_kernel_counter() requires an overflow handler, even though all we do is poll
 }

 static int gator_events_perf_pmu_read(int **buffer);

 static int gator_events_perf_pmu_online(int **buffer, bool migrate)
 {
 	return gator_events_perf_pmu_read(buffer);
 }

 static void __online_dispatch(int cpu, bool migrate, struct gator_attr *const attr, struct gator_event *const event)
 {
 	perf_overflow_handler_t handler;

 	event->zero = true;

 	if (event->pevent != NULL || event->pevent_attr == 0 || migrate) {
 		return;
 	}

 	if (attr->count > 0) {
 		handler = ebs_overflow_handler;
 	} else {
 		handler = dummy_handler;
 	}

 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 1, 0)
 	event->pevent = perf_event_create_kernel_counter(event->pevent_attr, cpu, 0, handler);
 #else
 	event->pevent = perf_event_create_kernel_counter(event->pevent_attr, cpu, 0, handler, 0);
 #endif
 	if (IS_ERR(event->pevent)) {
 		pr_debug("gator: unable to online a counter on cpu %d\n", cpu);
 		event->pevent = NULL;
 		return;
 	}

 	if (event->pevent->state != PERF_EVENT_STATE_ACTIVE) {
 		pr_debug("gator: inactive counter on cpu %d\n", cpu);
 		perf_event_release_kernel(event->pevent);
 		event->pevent = NULL;
 		return;
 	}
 }

 static void gator_events_perf_pmu_online_dispatch(int cpu, bool migrate)
 {
 	int cnt;

 	cpu = pcpu_to_lcpu(cpu);

 	for (cnt = 0; cnt < attr_count; cnt++) {
 		__online_dispatch(cpu, migrate, &attrs[cnt], &per_cpu(events, cpu)[cnt]);
 	}

 	if (cpu == 0) {
 		for (cnt = 0; cnt < uc_attr_count; cnt++) {
 			__online_dispatch(cpu, migrate, &uc_attrs[cnt], &uc_events[cnt]);
 		}
 	}
 }

 static void __offline_dispatch(int cpu, struct gator_event *const event)
 {
 	struct perf_event *pe = NULL;

 	if (event->pevent) {
 		pe = event->pevent;
 		event->pevent = NULL;
 	}

 	if (pe) {
 		perf_event_release_kernel(pe);
 	}
 }

 static void gator_events_perf_pmu_offline_dispatch(int cpu, bool migrate)
 {
 	int cnt;

 	if (migrate) {
 		return;
 	}
 	cpu = pcpu_to_lcpu(cpu);

 	for (cnt = 0; cnt < attr_count; cnt++) {
 		__offline_dispatch(cpu, &per_cpu(events, cpu)[cnt]);
 	}

 	if (cpu == 0) {
 		for (cnt = 0; cnt < uc_attr_count; cnt++) {
 			__offline_dispatch(cpu, &uc_events[cnt]);
 		}
 	}
 }

 static int __check_ebs(struct gator_attr *const attr)
 {
 	if (attr->count > 0) {
 		if (!event_based_sampling) {
 			event_based_sampling = true;
 		} else {
 			printk(KERN_WARNING "gator: Only one ebs counter is allowed\n");
 			return -1;
 		}
 	}

 	return 0;
 }

 static int __start(struct gator_attr *const attr, struct gator_event *const event)
 {
 	u32 size = sizeof(struct perf_event_attr);

 	event->pevent = NULL;
 	if (!attr->enabled) {	// Skip disabled counters
 		return 0;
 	}

 	event->prev = 0;
 	event->curr = 0;
 	event->prev_delta = 0;
 	event->pevent_attr = kmalloc(size, GFP_KERNEL);
 	if (!event->pevent_attr) {
 		gator_events_perf_pmu_stop();
 		return -1;
 	}

 	memset(event->pevent_attr, 0, size);
 	event->pevent_attr->type = attr->type;
 	event->pevent_attr->size = size;
 	event->pevent_attr->config = attr->event;
 	event->pevent_attr->sample_period = attr->count;
 	event->pevent_attr->pinned = 1;

 	return 0;
 }

 static int gator_events_perf_pmu_start(void)
 {
 	int cnt, cpu;

 	event_based_sampling = false;
 	for (cnt = 0; cnt < attr_count; cnt++) {
 		if (__check_ebs(&attrs[cnt]) != 0) {
 			return -1;
 		}
 	}

 	for (cnt = 0; cnt < uc_attr_count; cnt++) {
 		if (__check_ebs(&uc_attrs[cnt]) != 0) {
 			return -1;
 		}
 	}

 	for_each_present_cpu(cpu) {
 		for (cnt = 0; cnt < attr_count; cnt++) {
 			if (__start(&attrs[cnt], &per_cpu(events, cpu)[cnt]) != 0) {
 				return -1;
 			}
 		}
 	}

 	for (cnt = 0; cnt < uc_attr_count; cnt++) {
 		if (__start(&uc_attrs[cnt], &uc_events[cnt]) != 0) {
 			return -1;
 		}
 	}

 	return 0;
 }

 static void __event_stop(struct gator_event *const event)
 {
 	if (event->pevent_attr) {
 		kfree(event->pevent_attr);
 		event->pevent_attr = NULL;
 	}
 }

 static void __attr_stop(struct gator_attr *const attr)
 {
 	attr->enabled = 0;
 	attr->event = 0;
 	attr->count = 0;
 }

 static void gator_events_perf_pmu_stop(void)
 {
 	unsigned int cnt, cpu;

 	for_each_present_cpu(cpu) {
 		for (cnt = 0; cnt < attr_count; cnt++) {
 			__event_stop(&per_cpu(events, cpu)[cnt]);
 		}
 	}

 	for (cnt = 0; cnt < uc_attr_count; cnt++) {
 		__event_stop(&uc_events[cnt]);
 	}

 	for (cnt = 0; cnt < attr_count; cnt++) {
 		__attr_stop(&attrs[cnt]);
 	}

 	for (cnt = 0; cnt < uc_attr_count; cnt++) {
 		__attr_stop(&uc_attrs[cnt]);
 	}
 }

 static void __read(int *const len, int cpu, struct gator_attr *const attr, struct gator_event *const event)
 {
 	int delta;

 	struct perf_event *const ev = event->pevent;
 	if (ev != NULL && ev->state == PERF_EVENT_STATE_ACTIVE) {
 		/* After creating the perf counter in __online_dispatch, there
 		 * is a race condition between gator_events_perf_pmu_online and
 		 * gator_events_perf_pmu_read. So have
 		 * gator_events_perf_pmu_online call gator_events_perf_pmu_read
 		 * and in __read check to see if it's the first call after
 		 * __online_dispatch and if so, run the online code.
 		 */
 		if (event->zero) {
 			ev->pmu->read(ev);
 			event->prev = event->curr = local64_read(&ev->count);
 			event->prev_delta = 0;
 			per_cpu(perf_cnt, cpu)[(*len)++] = attr->key;
 			per_cpu(perf_cnt, cpu)[(*len)++] = 0;
 			event->zero = false;
 		} else {
 			ev->pmu->read(ev);
 			event->curr = local64_read(&ev->count);
 			delta = event->curr - event->prev;
 			if (delta != 0 || delta != event->prev_delta) {
 				event->prev_delta = delta;
 				event->prev = event->curr;
 				per_cpu(perf_cnt, cpu)[(*len)++] = attr->key;
 				if (delta < 0) {
 					delta *= -1;
 				}
 				per_cpu(perf_cnt, cpu)[(*len)++] = delta;
 			}
 		}
 	}
 }

 static int gator_events_perf_pmu_read(int **buffer)
 {
 	int cnt, len = 0;
 	const int cpu = get_logical_cpu();

 	for (cnt = 0; cnt < attr_count; cnt++) {
 		__read(&len, cpu, &attrs[cnt], &per_cpu(events, cpu)[cnt]);
 	}

 	if (cpu == 0) {
 		for (cnt = 0; cnt < uc_attr_count; cnt++) {
 			__read(&len, cpu, &uc_attrs[cnt], &uc_events[cnt]);
 		}
 	}

 	if (buffer) {
 		*buffer = per_cpu(perf_cnt, cpu);
 	}

 	return len;
 }

 static struct gator_interface gator_events_perf_pmu_interface = {
 	.create_files = gator_events_perf_pmu_create_files,
 	.start = gator_events_perf_pmu_start,
 	.stop = gator_events_perf_pmu_stop,
 	.online = gator_events_perf_pmu_online,
 	.online_dispatch = gator_events_perf_pmu_online_dispatch,
 	.offline_dispatch = gator_events_perf_pmu_offline_dispatch,
 	.read = gator_events_perf_pmu_read,
 };

 static void __attr_init(struct gator_attr *const attr)
 {
 	attr->name[0] = '\0';
 	attr->enabled = 0;
 	attr->type = 0;
 	attr->event = 0;
 	attr->count = 0;
 	attr->key = gator_events_get_key();
 }

 #ifdef CONFIG_OF

 static const struct of_device_id arm_cci_matches[] = {
 	{.compatible = "arm,cci-400" },
 	{},
 };

 static int probe_cci_revision(void)
 {
 	struct device_node *np;
 	struct resource res;
 	void __iomem *cci_ctrl_base;
 	int rev;
 	int ret = DEFAULT_CCI_REVISION;

 	np = of_find_matching_node(NULL, arm_cci_matches);
 	if (!np) {
 		return ret;
 	}

 	if (of_address_to_resource(np, 0, &res)) {
 		goto node_put;
 	}

 	cci_ctrl_base = ioremap(res.start, resource_size(&res));

 	rev = (readl_relaxed(cci_ctrl_base + 0xfe8) >> 4) & 0xf;

 	if (rev <= 4) {
 		ret = 0;
 	} else if (rev <= 6) {
 		ret = 1;
 	}

 	iounmap(cci_ctrl_base);

  node_put:
 	of_node_put(np);

 	return ret;
 }

 #else

 static int probe_cci_revision(void)
 {
 	return DEFAULT_CCI_REVISION;
 }

 #endif

 static void gator_events_perf_pmu_cci_init(const int type)
 {
 	int cnt;
 	const char *cci_name;

 	switch (probe_cci_revision()) {
 	case 0:
 		cci_name = "cci-400";
 		break;
 	case 1:
 		cci_name = "cci-400-r1";
 		break;
 	default:
 		pr_debug("gator: unrecognized cci-400 revision\n");
 		return;
 	}

 	snprintf(uc_attrs[uc_attr_count].name, sizeof(uc_attrs[uc_attr_count].name), "%s_ccnt", cci_name);
 	uc_attrs[uc_attr_count].type = type;
 	++uc_attr_count;

 	for (cnt = 0; cnt < CCI_400; ++cnt, ++uc_attr_count) {
 		struct gator_attr *const attr = &uc_attrs[uc_attr_count];
 		snprintf(attr->name, sizeof(attr->name), "%s_cnt%d", cci_name, cnt);
 		attr->type = type;
 	}
 }

 static void gator_events_perf_pmu_cpu_init(const struct gator_cpu *const gator_cpu, const int type)
 {
 	int cnt;

 	snprintf(attrs[attr_count].name, sizeof(attrs[attr_count].name), "%s_ccnt", gator_cpu->pmnc_name);
 	attrs[attr_count].type = type;
 	++attr_count;

 	for (cnt = 0; cnt < gator_cpu->pmnc_counters; ++cnt, ++attr_count) {
 		struct gator_attr *const attr = &attrs[attr_count];
 		snprintf(attr->name, sizeof(attr->name), "%s_cnt%d", gator_cpu->pmnc_name, cnt);
 		attr->type = type;
 	}
 }

 int gator_events_perf_pmu_init(void)
 {
 	struct perf_event_attr pea;
 	struct perf_event *pe;
 	const struct gator_cpu *gator_cpu;
 	int type;
 	int cpu;
 	int cnt;
 	bool found_cpu = false;

 	for (cnt = 0; cnt < CNTMAX; cnt++) {
 		__attr_init(&attrs[cnt]);
 	}
 	for (cnt = 0; cnt < UCCNT; cnt++) {
 		__attr_init(&uc_attrs[cnt]);
 	}

 	memset(&pea, 0, sizeof(pea));
 	pea.size = sizeof(pea);
 	pea.config = 0xFF;
 	attr_count = 0;
 	uc_attr_count = 0;
 	for (type = PERF_TYPE_MAX; type < 0x20; ++type) {
 		pea.type = type;

 		// A particular PMU may work on some but not all cores, so try on each core
 		pe = NULL;
 		for_each_present_cpu(cpu) {
 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 1, 0)
 			pe = perf_event_create_kernel_counter(&pea, cpu, 0, dummy_handler);
 #else
 			pe = perf_event_create_kernel_counter(&pea, cpu, 0, dummy_handler, 0);
 #endif
 			if (!IS_ERR(pe)) {
 				break;
 			}
 		}
 		// Assume that valid PMUs are contiguous
 		if (IS_ERR(pe)) {
 			break;
 		}

 		if (pe->pmu != NULL && type == pe->pmu->type) {
 			if (strcmp("CCI", pe->pmu->name) == 0 || strcmp("CCI_400", pe->pmu->name) == 0) {
 				gator_events_perf_pmu_cci_init(type);
 			} else if ((gator_cpu = gator_find_cpu_by_pmu_name(pe->pmu->name)) != NULL) {
 				found_cpu = true;
 				gator_events_perf_pmu_cpu_init(gator_cpu, type);
 			}
 			// Initialize gator_attrs for dynamic PMUs here
 		}

 		perf_event_release_kernel(pe);
 	}

 	if (!found_cpu) {
 		const struct gator_cpu *const gator_cpu = gator_find_cpu_by_cpuid(gator_cpuid());
 		if (gator_cpu == NULL) {
 			return -1;
 		}
 		gator_events_perf_pmu_cpu_init(gator_cpu, PERF_TYPE_RAW);
 	}

 	// Initialize gator_attrs for non-dynamic PMUs here

 	if (attr_count > CNTMAX) {
 		printk(KERN_ERR "gator: Too many perf counters\n");
 		return -1;
 	}

 	if (uc_attr_count > UCCNT) {
 		printk(KERN_ERR "gator: Too many perf uncore counters\n");
 		return -1;
 	}

 	return gator_events_install(&gator_events_perf_pmu_interface);
 }

 #endif
	/**
	* Copyright (C) ARM Limited 2010-2014. All rights reserved.
	*
	* This program is free software; you can redistribute it and/or modify
	* it under the terms of the GNU General Public License version 2 as
	* published by the Free Software Foundation.
	*/

	#include "gator.h"

	// gator_events_armvX.c is used for Linux 2.6.x
	#if GATOR_PERF_PMU_SUPPORT

	#include <linux/io.h>
	#ifdef CONFIG_OF
	#include <linux/of_address.h>
	#endif
	#include <linux/perf_event.h>
	#include <linux/slab.h>

	extern bool event_based_sampling;

	// Maximum number of per-core counters - currently reserves enough space for two full hardware PMUs for big.LITTLE
	#define CNTMAX 16
	#define CCI_400 4
	// Maximum number of uncore counters
	// + 1 for the cci-400 cycles counter
	#define UCCNT (CCI_400 + 1)

	// Default to 0 if unable to probe the revision which was the previous behavior
	#define DEFAULT_CCI_REVISION 0

	// A gator_attr is needed for every counter
	struct gator_attr {
	// Set once in gator_events_perf_pmu_*_init - the name of the event in the gatorfs
	char name[40];
	// Exposed in gatorfs - set by gatord to enable this counter
	unsigned long enabled;
	// Set once in gator_events_perf_pmu_*_init - the perf type to use, see perf_type_id in the perf_event.h header file.
	unsigned long type;
	// Exposed in gatorfs - set by gatord to select the event to collect
	unsigned long event;
	// Exposed in gatorfs - set by gatord with the sample period to use and enable EBS for this counter
	unsigned long count;
	// Exposed as read only in gatorfs - set once in __attr_init as the key to use in the APC data
	unsigned long key;
	};

	// Per-core counter attributes
	static struct gator_attr attrs[CNTMAX];
	// Number of initialized per-core counters
	static int attr_count;
	// Uncore counter attributes
	static struct gator_attr uc_attrs[UCCNT];
	// Number of initialized uncore counters
	static int uc_attr_count;

	struct gator_event {
	int curr;
	int prev;
	int prev_delta;
	bool zero;
	struct perf_event *pevent;
	struct perf_event_attr *pevent_attr;
	};

	static DEFINE_PER_CPU(struct gator_event[CNTMAX], events);
	static struct gator_event uc_events[UCCNT];
	static DEFINE_PER_CPU(int[(CNTMAX + UCCNT)*2], perf_cnt);

	static void gator_events_perf_pmu_stop(void);

	static int __create_files(struct super_block sb, struct dentry root, struct gator_attr *const attr)
	{
	struct dentry *dir;

	if (attr->name[0] == '\0') {
	return 0;
	}
	dir = gatorfs_mkdir(sb, root, attr->name);
	if (!dir) {
	return -1;
	}
	gatorfs_create_ulong(sb, dir, "enabled", &attr->enabled);
	gatorfs_create_ulong(sb, dir, "count", &attr->count);
	gatorfs_create_ro_ulong(sb, dir, "key", &attr->key);
	gatorfs_create_ulong(sb, dir, "event", &attr->event);

	return 0;
	}

	static int gator_events_perf_pmu_create_files(struct super_block sb, struct dentry root)
	{
	int cnt;

	for (cnt = 0; cnt < attr_count; cnt++) {
	if (__create_files(sb, root, &attrs[cnt]) != 0) {
	return -1;
	}
	}

	for (cnt = 0; cnt < uc_attr_count; cnt++) {
	if (__create_files(sb, root, &uc_attrs[cnt]) != 0) {
	return -1;
	}
	}

	return 0;
	}

	#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 1, 0)
	static void ebs_overflow_handler(struct perf_event event, int unused, struct perf_sample_data data, struct pt_regs *regs)
	#else
	static void ebs_overflow_handler(struct perf_event event, struct perf_sample_data data, struct pt_regs *regs)
	#endif
	{
	gator_backtrace_handler(regs);
	}

	#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 1, 0)
	static void dummy_handler(struct perf_event event, int unused, struct perf_sample_data data, struct pt_regs *regs)
	#else
	static void dummy_handler(struct perf_event event, struct perf_sample_data data, struct pt_regs *regs)
	#endif
	{
	// Required as perf_event_create_kernel_counter() requires an overflow handler, even though all we do is poll
	}

	static int gator_events_perf_pmu_read(int **buffer);

	static int gator_events_perf_pmu_online(int **buffer, bool migrate)
	{
	return gator_events_perf_pmu_read(buffer);
	}

	static void __online_dispatch(int cpu, bool migrate, struct gator_attr const attr, struct gator_event const event)
	{
	perf_overflow_handler_t handler;

	event->zero = true;

	if (event->pevent != NULL \|\| event->pevent_attr == 0 \|\| migrate) {
	return;
	}

	if (attr->count > 0) {
	handler = ebs_overflow_handler;
	} else {
	handler = dummy_handler;
	}

	#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 1, 0)
	event->pevent = perf_event_create_kernel_counter(event->pevent_attr, cpu, 0, handler);
	#else
	event->pevent = perf_event_create_kernel_counter(event->pevent_attr, cpu, 0, handler, 0);
	#endif
	if (IS_ERR(event->pevent)) {
	pr_debug("gator: unable to online a counter on cpu %d\n", cpu);
	event->pevent = NULL;
	return;
	}

	if (event->pevent->state != PERF_EVENT_STATE_ACTIVE) {
	pr_debug("gator: inactive counter on cpu %d\n", cpu);
	perf_event_release_kernel(event->pevent);
	event->pevent = NULL;
	return;
	}
	}

	static void gator_events_perf_pmu_online_dispatch(int cpu, bool migrate)
	{
	int cnt;

	cpu = pcpu_to_lcpu(cpu);

	for (cnt = 0; cnt < attr_count; cnt++) {
	__online_dispatch(cpu, migrate, &attrs[cnt], &per_cpu(events, cpu)[cnt]);
	}

	if (cpu == 0) {
	for (cnt = 0; cnt < uc_attr_count; cnt++) {
	__online_dispatch(cpu, migrate, &uc_attrs[cnt], &uc_events[cnt]);
	}
	}
	}

	static void __offline_dispatch(int cpu, struct gator_event *const event)
	{
	struct perf_event *pe = NULL;

	if (event->pevent) {
	pe = event->pevent;
	event->pevent = NULL;
	}

	if (pe) {
	perf_event_release_kernel(pe);
	}
	}

	static void gator_events_perf_pmu_offline_dispatch(int cpu, bool migrate)
	{
	int cnt;

	if (migrate) {
	return;
	}
	cpu = pcpu_to_lcpu(cpu);

	for (cnt = 0; cnt < attr_count; cnt++) {
	__offline_dispatch(cpu, &per_cpu(events, cpu)[cnt]);
	}

	if (cpu == 0) {
	for (cnt = 0; cnt < uc_attr_count; cnt++) {
	__offline_dispatch(cpu, &uc_events[cnt]);
	}
	}
	}

	static int __check_ebs(struct gator_attr *const attr)
	{
	if (attr->count > 0) {
	if (!event_based_sampling) {
	event_based_sampling = true;
	} else {
	printk(KERN_WARNING "gator: Only one ebs counter is allowed\n");
	return -1;
	}
	}

	return 0;
	}

	static int __start(struct gator_attr const attr, struct gator_event const event)
	{
	u32 size = sizeof(struct perf_event_attr);

	event->pevent = NULL;
	if (!attr->enabled) { // Skip disabled counters
	return 0;
	}

	event->prev = 0;
	event->curr = 0;
	event->prev_delta = 0;
	event->pevent_attr = kmalloc(size, GFP_KERNEL);
	if (!event->pevent_attr) {
	gator_events_perf_pmu_stop();
	return -1;
	}

	memset(event->pevent_attr, 0, size);
	event->pevent_attr->type = attr->type;
	event->pevent_attr->size = size;
	event->pevent_attr->config = attr->event;
	event->pevent_attr->sample_period = attr->count;
	event->pevent_attr->pinned = 1;

	return 0;
	}

	static int gator_events_perf_pmu_start(void)
	{
	int cnt, cpu;

	event_based_sampling = false;
	for (cnt = 0; cnt < attr_count; cnt++) {
	if (__check_ebs(&attrs[cnt]) != 0) {
	return -1;
	}
	}

	for (cnt = 0; cnt < uc_attr_count; cnt++) {
	if (__check_ebs(&uc_attrs[cnt]) != 0) {
	return -1;
	}
	}

	for_each_present_cpu(cpu) {
	for (cnt = 0; cnt < attr_count; cnt++) {
	if (__start(&attrs[cnt], &per_cpu(events, cpu)[cnt]) != 0) {
	return -1;
	}
	}
	}

	for (cnt = 0; cnt < uc_attr_count; cnt++) {
	if (__start(&uc_attrs[cnt], &uc_events[cnt]) != 0) {
	return -1;
	}
	}

	return 0;
	}

	static void __event_stop(struct gator_event *const event)
	{
	if (event->pevent_attr) {
	kfree(event->pevent_attr);
	event->pevent_attr = NULL;
	}
	}

	static void __attr_stop(struct gator_attr *const attr)
	{
	attr->enabled = 0;
	attr->event = 0;
	attr->count = 0;
	}

	static void gator_events_perf_pmu_stop(void)
	{
	unsigned int cnt, cpu;

	for_each_present_cpu(cpu) {
	for (cnt = 0; cnt < attr_count; cnt++) {
	__event_stop(&per_cpu(events, cpu)[cnt]);
	}
	}

	for (cnt = 0; cnt < uc_attr_count; cnt++) {
	__event_stop(&uc_events[cnt]);
	}

	for (cnt = 0; cnt < attr_count; cnt++) {
	__attr_stop(&attrs[cnt]);
	}

	for (cnt = 0; cnt < uc_attr_count; cnt++) {
	__attr_stop(&uc_attrs[cnt]);
	}
	}

	static void __read(int const len, int cpu, struct gator_attr const attr, struct gator_event *const event)
	{
	int delta;

	struct perf_event *const ev = event->pevent;
	if (ev != NULL && ev->state == PERF_EVENT_STATE_ACTIVE) {
	/* After creating the perf counter in __online_dispatch, there
	* is a race condition between gator_events_perf_pmu_online and
	* gator_events_perf_pmu_read. So have
	* gator_events_perf_pmu_online call gator_events_perf_pmu_read
	* and in __read check to see if it's the first call after
	* __online_dispatch and if so, run the online code.
	*/
	if (event->zero) {
	ev->pmu->read(ev);
	event->prev = event->curr = local64_read(&ev->count);
	event->prev_delta = 0;
	per_cpu(perf_cnt, cpu)[(*len)++] = attr->key;
	per_cpu(perf_cnt, cpu)[(*len)++] = 0;
	event->zero = false;
	} else {
	ev->pmu->read(ev);
	event->curr = local64_read(&ev->count);
	delta = event->curr - event->prev;
	if (delta != 0 \|\| delta != event->prev_delta) {
	event->prev_delta = delta;
	event->prev = event->curr;
	per_cpu(perf_cnt, cpu)[(*len)++] = attr->key;
	if (delta < 0) {
	delta *= -1;
	}
	per_cpu(perf_cnt, cpu)[(*len)++] = delta;
	}
	}
	}
	}

	static int gator_events_perf_pmu_read(int **buffer)
	{
	int cnt, len = 0;
	const int cpu = get_logical_cpu();

	for (cnt = 0; cnt < attr_count; cnt++) {
	__read(&len, cpu, &attrs[cnt], &per_cpu(events, cpu)[cnt]);
	}

	if (cpu == 0) {
	for (cnt = 0; cnt < uc_attr_count; cnt++) {
	__read(&len, cpu, &uc_attrs[cnt], &uc_events[cnt]);
	}
	}

	if (buffer) {
	*buffer = per_cpu(perf_cnt, cpu);
	}

	return len;
	}

	static struct gator_interface gator_events_perf_pmu_interface = {
	.create_files = gator_events_perf_pmu_create_files,
	.start = gator_events_perf_pmu_start,
	.stop = gator_events_perf_pmu_stop,
	.online = gator_events_perf_pmu_online,
	.online_dispatch = gator_events_perf_pmu_online_dispatch,
	.offline_dispatch = gator_events_perf_pmu_offline_dispatch,
	.read = gator_events_perf_pmu_read,
	};

	static void __attr_init(struct gator_attr *const attr)
	{
	attr->name[0] = '\0';
	attr->enabled = 0;
	attr->type = 0;
	attr->event = 0;
	attr->count = 0;
	attr->key = gator_events_get_key();
	}

	#ifdef CONFIG_OF

	static const struct of_device_id arm_cci_matches[] = {
	{.compatible = "arm,cci-400" },
	{},
	};

	static int probe_cci_revision(void)
	{
	struct device_node *np;
	struct resource res;
	void __iomem *cci_ctrl_base;
	int rev;
	int ret = DEFAULT_CCI_REVISION;

	np = of_find_matching_node(NULL, arm_cci_matches);
	if (!np) {
	return ret;
	}

	if (of_address_to_resource(np, 0, &res)) {
	goto node_put;
	}

	cci_ctrl_base = ioremap(res.start, resource_size(&res));

	rev = (readl_relaxed(cci_ctrl_base + 0xfe8) >> 4) & 0xf;

	if (rev <= 4) {
	ret = 0;
	} else if (rev <= 6) {
	ret = 1;
	}

	iounmap(cci_ctrl_base);

	node_put:
	of_node_put(np);

	return ret;
	}

	#else

	static int probe_cci_revision(void)
	{
	return DEFAULT_CCI_REVISION;
	}

	#endif

	static void gator_events_perf_pmu_cci_init(const int type)
	{
	int cnt;
	const char *cci_name;

	switch (probe_cci_revision()) {
	case 0:
	cci_name = "cci-400";
	break;
	case 1:
	cci_name = "cci-400-r1";
	break;
	default:
	pr_debug("gator: unrecognized cci-400 revision\n");
	return;
	}

	snprintf(uc_attrs[uc_attr_count].name, sizeof(uc_attrs[uc_attr_count].name), "%s_ccnt", cci_name);
	uc_attrs[uc_attr_count].type = type;
	++uc_attr_count;

	for (cnt = 0; cnt < CCI_400; ++cnt, ++uc_attr_count) {
	struct gator_attr *const attr = &uc_attrs[uc_attr_count];
	snprintf(attr->name, sizeof(attr->name), "%s_cnt%d", cci_name, cnt);
	attr->type = type;
	}
	}

	static void gator_events_perf_pmu_cpu_init(const struct gator_cpu *const gator_cpu, const int type)
	{
	int cnt;

	snprintf(attrs[attr_count].name, sizeof(attrs[attr_count].name), "%s_ccnt", gator_cpu->pmnc_name);
	attrs[attr_count].type = type;
	++attr_count;

	for (cnt = 0; cnt < gator_cpu->pmnc_counters; ++cnt, ++attr_count) {
	struct gator_attr *const attr = &attrs[attr_count];
	snprintf(attr->name, sizeof(attr->name), "%s_cnt%d", gator_cpu->pmnc_name, cnt);
	attr->type = type;
	}
	}

	int gator_events_perf_pmu_init(void)
	{
	struct perf_event_attr pea;
	struct perf_event *pe;
	const struct gator_cpu *gator_cpu;
	int type;
	int cpu;
	int cnt;
	bool found_cpu = false;

	for (cnt = 0; cnt < CNTMAX; cnt++) {
	__attr_init(&attrs[cnt]);
	}
	for (cnt = 0; cnt < UCCNT; cnt++) {
	__attr_init(&uc_attrs[cnt]);
	}

	memset(&pea, 0, sizeof(pea));
	pea.size = sizeof(pea);
	pea.config = 0xFF;
	attr_count = 0;
	uc_attr_count = 0;
	for (type = PERF_TYPE_MAX; type < 0x20; ++type) {
	pea.type = type;

	// A particular PMU may work on some but not all cores, so try on each core
	pe = NULL;
	for_each_present_cpu(cpu) {
	#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 1, 0)
	pe = perf_event_create_kernel_counter(&pea, cpu, 0, dummy_handler);
	#else
	pe = perf_event_create_kernel_counter(&pea, cpu, 0, dummy_handler, 0);
	#endif
	if (!IS_ERR(pe)) {
	break;
	}
	}
	// Assume that valid PMUs are contiguous
	if (IS_ERR(pe)) {
	break;
	}

	if (pe->pmu != NULL && type == pe->pmu->type) {
	if (strcmp("CCI", pe->pmu->name) == 0 \|\| strcmp("CCI_400", pe->pmu->name) == 0) {
	gator_events_perf_pmu_cci_init(type);
	} else if ((gator_cpu = gator_find_cpu_by_pmu_name(pe->pmu->name)) != NULL) {
	found_cpu = true;
	gator_events_perf_pmu_cpu_init(gator_cpu, type);
	}
	// Initialize gator_attrs for dynamic PMUs here
	}

	perf_event_release_kernel(pe);
	}

	if (!found_cpu) {
	const struct gator_cpu *const gator_cpu = gator_find_cpu_by_cpuid(gator_cpuid());
	if (gator_cpu == NULL) {
	return -1;
	}
	gator_events_perf_pmu_cpu_init(gator_cpu, PERF_TYPE_RAW);
	}

	// Initialize gator_attrs for non-dynamic PMUs here

	if (attr_count > CNTMAX) {
	printk(KERN_ERR "gator: Too many perf counters\n");
	return -1;
	}

	if (uc_attr_count > UCCNT) {
	printk(KERN_ERR "gator: Too many perf uncore counters\n");
	return -1;
	}

	return gator_events_install(&gator_events_perf_pmu_interface);
	}

	#endif