kernel/rcutiny_plugin.h - arm/linux-arm64-legacy - Git at Google

 /*
  * Read-Copy Update mechanism for mutual exclusion, the Bloatwatch edition
  * Internal non-public definitions that provide either classic
  * or preemptible semantics.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
  * the Free Software Foundation; either version 2 of the License, or
  * (at your option) any later version.
  *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  *
  * Copyright (c) 2010 Linaro
  *
  * Author: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
  */

 #include <linux/kthread.h>
 #include <linux/module.h>
 #include <linux/debugfs.h>
 #include <linux/seq_file.h>

 /* Global control variables for rcupdate callback mechanism. */
 struct rcu_ctrlblk {
 	struct rcu_head *rcucblist;	/* List of pending callbacks (CBs). */
 	struct rcu_head **donetail;	/* ->next pointer of last "done" CB. */
 	struct rcu_head **curtail;	/* ->next pointer of last CB. */
 	RCU_TRACE(long qlen);		/* Number of pending CBs. */
 	RCU_TRACE(unsigned long gp_start); /* Start time for stalls. */
 	RCU_TRACE(unsigned long ticks_this_gp); /* Statistic for stalls. */
 	RCU_TRACE(unsigned long jiffies_stall); /* Jiffies at next stall. */
 	RCU_TRACE(char *name);		/* Name of RCU type. */
 };

 /* Definition for rcupdate control block. */
 static struct rcu_ctrlblk rcu_sched_ctrlblk = {
 	.donetail	= &rcu_sched_ctrlblk.rcucblist,
 	.curtail	= &rcu_sched_ctrlblk.rcucblist,
 	RCU_TRACE(.name = "rcu_sched")
 };

 static struct rcu_ctrlblk rcu_bh_ctrlblk = {
 	.donetail	= &rcu_bh_ctrlblk.rcucblist,
 	.curtail	= &rcu_bh_ctrlblk.rcucblist,
 	RCU_TRACE(.name = "rcu_bh")
 };

 #ifdef CONFIG_DEBUG_LOCK_ALLOC
 int rcu_scheduler_active __read_mostly;
 EXPORT_SYMBOL_GPL(rcu_scheduler_active);
 #endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */

 #ifdef CONFIG_RCU_TRACE

 static void check_cpu_stall(struct rcu_ctrlblk *rcp)
 {
 	unsigned long j;
 	unsigned long js;

 	if (rcu_cpu_stall_suppress)
 		return;
 	rcp->ticks_this_gp++;
 	j = jiffies;
 	js = rcp->jiffies_stall;
 	if (*rcp->curtail && ULONG_CMP_GE(j, js)) {
 		pr_err("INFO: %s stall on CPU (%lu ticks this GP) idle=%llx (t=%lu jiffies q=%ld)\n",
 		       rcp->name, rcp->ticks_this_gp, rcu_dynticks_nesting,
 		       jiffies - rcp->gp_start, rcp->qlen);
 		dump_stack();
 	}
 	if (*rcp->curtail && ULONG_CMP_GE(j, js))
 		rcp->jiffies_stall = jiffies +
 			3 * rcu_jiffies_till_stall_check() + 3;
 	else if (ULONG_CMP_GE(j, js))
 		rcp->jiffies_stall = jiffies + rcu_jiffies_till_stall_check();
 }

 static void check_cpu_stall_preempt(void);

 #endif /* #ifdef CONFIG_RCU_TRACE */

 static void reset_cpu_stall_ticks(struct rcu_ctrlblk *rcp)
 {
 #ifdef CONFIG_RCU_TRACE
 	rcp->ticks_this_gp = 0;
 	rcp->gp_start = jiffies;
 	rcp->jiffies_stall = jiffies + rcu_jiffies_till_stall_check();
 #endif /* #ifdef CONFIG_RCU_TRACE */
 }

 static void check_cpu_stalls(void)
 {
 	RCU_TRACE(check_cpu_stall(&rcu_bh_ctrlblk));
 	RCU_TRACE(check_cpu_stall(&rcu_sched_ctrlblk));
 	RCU_TRACE(check_cpu_stall_preempt());
 }

 #ifdef CONFIG_RCU_TRACE

 /*
  * Because preemptible RCU does not exist, it is not necessary to
  * dump out its statistics.
  */
 static void show_tiny_preempt_stats(struct seq_file *m)
 {
 }

 #endif /* #ifdef CONFIG_RCU_TRACE */

 /*
  * Because preemptible RCU does not exist, it never has any callbacks
  * to check.
  */
 static void rcu_preempt_check_callbacks(void)
 {
 }

 /*
  * Because preemptible RCU does not exist, it never has any callbacks
  * to remove.
  */
 static void rcu_preempt_remove_callbacks(struct rcu_ctrlblk *rcp)
 {
 }

 /*
  * Because preemptible RCU does not exist, it never has any callbacks
  * to process.
  */
 static void rcu_preempt_process_callbacks(void)
 {
 }

 /* Hold off callback invocation until early_initcall() time. */
 static int rcu_scheduler_fully_active __read_mostly;

 /*
  * Start up softirq processing of callbacks.
  */
 void invoke_rcu_callbacks(void)
 {
 	if (rcu_scheduler_fully_active)
 		raise_softirq(RCU_SOFTIRQ);
 }

 #ifdef CONFIG_RCU_TRACE

 /*
  * There is no callback kthread, so this thread is never it.
  */
 static bool rcu_is_callbacks_kthread(void)
 {
 	return false;
 }

 #endif /* #ifdef CONFIG_RCU_TRACE */

 static int __init rcu_scheduler_really_started(void)
 {
 	rcu_scheduler_fully_active = 1;
 	open_softirq(RCU_SOFTIRQ, rcu_process_callbacks);
 	raise_softirq(RCU_SOFTIRQ);  /* Invoke any callbacks from early boot. */
 	return 0;
 }
 early_initcall(rcu_scheduler_really_started);

 #ifdef CONFIG_DEBUG_LOCK_ALLOC
 #include <linux/kernel_stat.h>

 /*
  * During boot, we forgive RCU lockdep issues.  After this function is
  * invoked, we start taking RCU lockdep issues seriously.
  */
 void __init rcu_scheduler_starting(void)
 {
 	WARN_ON(nr_context_switches() > 0);
 	rcu_scheduler_active = 1;
 }

 #endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */

 #ifdef CONFIG_RCU_TRACE

 static void rcu_trace_sub_qlen(struct rcu_ctrlblk *rcp, int n)
 {
 	unsigned long flags;

 	local_irq_save(flags);
 	rcp->qlen -= n;
 	local_irq_restore(flags);
 }

 /*
  * Dump statistics for TINY_RCU, such as they are.
  */
 static int show_tiny_stats(struct seq_file *m, void *unused)
 {
 	show_tiny_preempt_stats(m);
 	seq_printf(m, "rcu_sched: qlen: %ld\n", rcu_sched_ctrlblk.qlen);
 	seq_printf(m, "rcu_bh: qlen: %ld\n", rcu_bh_ctrlblk.qlen);
 	return 0;
 }

 static int show_tiny_stats_open(struct inode *inode, struct file *file)
 {
 	return single_open(file, show_tiny_stats, NULL);
 }

 static const struct file_operations show_tiny_stats_fops = {
 	.owner = THIS_MODULE,
 	.open = show_tiny_stats_open,
 	.read = seq_read,
 	.llseek = seq_lseek,
 	.release = single_release,
 };

 static struct dentry *rcudir;

 static int __init rcutiny_trace_init(void)
 {
 	struct dentry *retval;

 	rcudir = debugfs_create_dir("rcu", NULL);
 	if (!rcudir)
 		goto free_out;
 	retval = debugfs_create_file("rcudata", 0444, rcudir,
 				     NULL, &show_tiny_stats_fops);
 	if (!retval)
 		goto free_out;
 	return 0;
 free_out:
 	debugfs_remove_recursive(rcudir);
 	return 1;
 }

 static void __exit rcutiny_trace_cleanup(void)
 {
 	debugfs_remove_recursive(rcudir);
 }

 module_init(rcutiny_trace_init);
 module_exit(rcutiny_trace_cleanup);

 MODULE_AUTHOR("Paul E. McKenney");
 MODULE_DESCRIPTION("Read-Copy Update tracing for tiny implementation");
 MODULE_LICENSE("GPL");

 static void check_cpu_stall_preempt(void)
 {
 }

 #endif /* #ifdef CONFIG_RCU_TRACE */
	/*
	* Read-Copy Update mechanism for mutual exclusion, the Bloatwatch edition
	* Internal non-public definitions that provide either classic
	* or preemptible semantics.
	*
	* This program is free software; you can redistribute it and/or modify
	* it under the terms of the GNU General Public License as published by
	* the Free Software Foundation; either version 2 of the License, or
	* (at your option) any later version.
	*
	* This program is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	* GNU General Public License for more details.
	*
	* You should have received a copy of the GNU General Public License
	* along with this program; if not, write to the Free Software
	* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
	*
	* Copyright (c) 2010 Linaro
	*
	* Author: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
	*/

	#include <linux/kthread.h>
	#include <linux/module.h>
	#include <linux/debugfs.h>
	#include <linux/seq_file.h>

	/* Global control variables for rcupdate callback mechanism. */
	struct rcu_ctrlblk {
	struct rcu_head rcucblist; / List of pending callbacks (CBs). */
	struct rcu_head *donetail; / ->next pointer of last "done" CB. */
	struct rcu_head *curtail; / ->next pointer of last CB. */
	RCU_TRACE(long qlen); /* Number of pending CBs. */
	RCU_TRACE(unsigned long gp_start); /* Start time for stalls. */
	RCU_TRACE(unsigned long ticks_this_gp); /* Statistic for stalls. */
	RCU_TRACE(unsigned long jiffies_stall); /* Jiffies at next stall. */
	RCU_TRACE(char name); / Name of RCU type. */
	};

	/* Definition for rcupdate control block. */
	static struct rcu_ctrlblk rcu_sched_ctrlblk = {
	.donetail = &rcu_sched_ctrlblk.rcucblist,
	.curtail = &rcu_sched_ctrlblk.rcucblist,
	RCU_TRACE(.name = "rcu_sched")
	};

	static struct rcu_ctrlblk rcu_bh_ctrlblk = {
	.donetail = &rcu_bh_ctrlblk.rcucblist,
	.curtail = &rcu_bh_ctrlblk.rcucblist,
	RCU_TRACE(.name = "rcu_bh")
	};

	#ifdef CONFIG_DEBUG_LOCK_ALLOC
	int rcu_scheduler_active __read_mostly;
	EXPORT_SYMBOL_GPL(rcu_scheduler_active);
	#endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */

	#ifdef CONFIG_RCU_TRACE

	static void check_cpu_stall(struct rcu_ctrlblk *rcp)
	{
	unsigned long j;
	unsigned long js;

	if (rcu_cpu_stall_suppress)
	return;
	rcp->ticks_this_gp++;
	j = jiffies;
	js = rcp->jiffies_stall;
	if (*rcp->curtail && ULONG_CMP_GE(j, js)) {
	pr_err("INFO: %s stall on CPU (%lu ticks this GP) idle=%llx (t=%lu jiffies q=%ld)\n",
	rcp->name, rcp->ticks_this_gp, rcu_dynticks_nesting,
	jiffies - rcp->gp_start, rcp->qlen);
	dump_stack();
	}
	if (*rcp->curtail && ULONG_CMP_GE(j, js))
	rcp->jiffies_stall = jiffies +
	3 * rcu_jiffies_till_stall_check() + 3;
	else if (ULONG_CMP_GE(j, js))
	rcp->jiffies_stall = jiffies + rcu_jiffies_till_stall_check();
	}

	static void check_cpu_stall_preempt(void);

	#endif /* #ifdef CONFIG_RCU_TRACE */

	static void reset_cpu_stall_ticks(struct rcu_ctrlblk *rcp)
	{
	#ifdef CONFIG_RCU_TRACE
	rcp->ticks_this_gp = 0;
	rcp->gp_start = jiffies;
	rcp->jiffies_stall = jiffies + rcu_jiffies_till_stall_check();
	#endif /* #ifdef CONFIG_RCU_TRACE */
	}

	static void check_cpu_stalls(void)
	{
	RCU_TRACE(check_cpu_stall(&rcu_bh_ctrlblk));
	RCU_TRACE(check_cpu_stall(&rcu_sched_ctrlblk));
	RCU_TRACE(check_cpu_stall_preempt());
	}

	#ifdef CONFIG_RCU_TRACE

	/*
	* Because preemptible RCU does not exist, it is not necessary to
	* dump out its statistics.
	*/
	static void show_tiny_preempt_stats(struct seq_file *m)
	{
	}

	#endif /* #ifdef CONFIG_RCU_TRACE */

	/*
	* Because preemptible RCU does not exist, it never has any callbacks
	* to check.
	*/
	static void rcu_preempt_check_callbacks(void)
	{
	}

	/*
	* Because preemptible RCU does not exist, it never has any callbacks
	* to remove.
	*/
	static void rcu_preempt_remove_callbacks(struct rcu_ctrlblk *rcp)
	{
	}

	/*
	* Because preemptible RCU does not exist, it never has any callbacks
	* to process.
	*/
	static void rcu_preempt_process_callbacks(void)
	{
	}

	/* Hold off callback invocation until early_initcall() time. */
	static int rcu_scheduler_fully_active __read_mostly;

	/*
	* Start up softirq processing of callbacks.
	*/
	void invoke_rcu_callbacks(void)
	{
	if (rcu_scheduler_fully_active)
	raise_softirq(RCU_SOFTIRQ);
	}

	#ifdef CONFIG_RCU_TRACE

	/*
	* There is no callback kthread, so this thread is never it.
	*/
	static bool rcu_is_callbacks_kthread(void)
	{
	return false;
	}

	#endif /* #ifdef CONFIG_RCU_TRACE */

	static int __init rcu_scheduler_really_started(void)
	{
	rcu_scheduler_fully_active = 1;
	open_softirq(RCU_SOFTIRQ, rcu_process_callbacks);
	raise_softirq(RCU_SOFTIRQ); /* Invoke any callbacks from early boot. */
	return 0;
	}
	early_initcall(rcu_scheduler_really_started);

	#ifdef CONFIG_DEBUG_LOCK_ALLOC
	#include <linux/kernel_stat.h>

	/*
	* During boot, we forgive RCU lockdep issues. After this function is
	* invoked, we start taking RCU lockdep issues seriously.
	*/
	void __init rcu_scheduler_starting(void)
	{
	WARN_ON(nr_context_switches() > 0);
	rcu_scheduler_active = 1;
	}

	#endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */

	#ifdef CONFIG_RCU_TRACE

	static void rcu_trace_sub_qlen(struct rcu_ctrlblk *rcp, int n)
	{
	unsigned long flags;

	local_irq_save(flags);
	rcp->qlen -= n;
	local_irq_restore(flags);
	}

	/*
	* Dump statistics for TINY_RCU, such as they are.
	*/
	static int show_tiny_stats(struct seq_file m, void unused)
	{
	show_tiny_preempt_stats(m);
	seq_printf(m, "rcu_sched: qlen: %ld\n", rcu_sched_ctrlblk.qlen);
	seq_printf(m, "rcu_bh: qlen: %ld\n", rcu_bh_ctrlblk.qlen);
	return 0;
	}

	static int show_tiny_stats_open(struct inode inode, struct file file)
	{
	return single_open(file, show_tiny_stats, NULL);
	}

	static const struct file_operations show_tiny_stats_fops = {
	.owner = THIS_MODULE,
	.open = show_tiny_stats_open,
	.read = seq_read,
	.llseek = seq_lseek,
	.release = single_release,
	};

	static struct dentry *rcudir;

	static int __init rcutiny_trace_init(void)
	{
	struct dentry *retval;

	rcudir = debugfs_create_dir("rcu", NULL);
	if (!rcudir)
	goto free_out;
	retval = debugfs_create_file("rcudata", 0444, rcudir,
	NULL, &show_tiny_stats_fops);
	if (!retval)
	goto free_out;
	return 0;
	free_out:
	debugfs_remove_recursive(rcudir);
	return 1;
	}

	static void __exit rcutiny_trace_cleanup(void)
	{
	debugfs_remove_recursive(rcudir);
	}

	module_init(rcutiny_trace_init);
	module_exit(rcutiny_trace_cleanup);

	MODULE_AUTHOR("Paul E. McKenney");
	MODULE_DESCRIPTION("Read-Copy Update tracing for tiny implementation");
	MODULE_LICENSE("GPL");

	static void check_cpu_stall_preempt(void)
	{
	}

	#endif /* #ifdef CONFIG_RCU_TRACE */