arch/powerpc/mm/icswx.c - arm/linux - Git at Google

 /*
  *  ICSWX and ACOP Management
  *
  *  Copyright (C) 2011 Anton Blanchard, IBM Corp. <anton@samba.org>
  *
  *  This program is free software; you can redistribute it and/or
  *  modify it under the terms of the GNU General Public License
  *  as published by the Free Software Foundation; either version
  *  2 of the License, or (at your option) any later version.
  *
  */

 #include <linux/sched.h>
 #include <linux/kernel.h>
 #include <linux/errno.h>
 #include <linux/types.h>
 #include <linux/mm.h>
 #include <linux/spinlock.h>
 #include <linux/module.h>
 #include <linux/uaccess.h>

 #include "icswx.h"

 /*
  * The processor and its L2 cache cause the icswx instruction to
  * generate a COP_REQ transaction on PowerBus. The transaction has no
  * address, and the processor does not perform an MMU access to
  * authenticate the transaction. The command portion of the PowerBus
  * COP_REQ transaction includes the LPAR_ID (LPID) and the coprocessor
  * Process ID (PID), which the coprocessor compares to the authorized
  * LPID and PID held in the coprocessor, to determine if the process
  * is authorized to generate the transaction.  The data of the COP_REQ
  * transaction is 128-byte or less in size and is placed in cacheable
  * memory on a 128-byte cache line boundary.
  *
  * The task to use a coprocessor should use use_cop() to mark the use
  * of the Coprocessor Type (CT) and context switching. On a server
  * class processor, the PID register is used only for coprocessor
  * management + * and so a coprocessor PID is allocated before
  * executing icswx + * instruction. Drop_cop() is used to free the
  * coprocessor PID.
  *
  * Example:
  * Host Fabric Interface (HFI) is a PowerPC network coprocessor.
  * Each HFI have multiple windows. Each HFI window serves as a
  * network device sending to and receiving from HFI network.
  * HFI immediate send function uses icswx instruction. The immediate
  * send function allows small (single cache-line) packets be sent
  * without using the regular HFI send FIFO and doorbell, which are
  * much slower than immediate send.
  *
  * For each task intending to use HFI immediate send, the HFI driver
  * calls use_cop() to obtain a coprocessor PID for the task.
  * The HFI driver then allocate a free HFI window and save the
  * coprocessor PID to the HFI window to allow the task to use the
  * HFI window.
  *
  * The HFI driver repeatedly creates immediate send packets and
  * issues icswx instruction to send data through the HFI window.
  * The HFI compares the coprocessor PID in the CPU PID register
  * to the PID held in the HFI window to determine if the transaction
  * is allowed.
  *
  * When the task to release the HFI window, the HFI driver calls
  * drop_cop() to release the coprocessor PID.
  */

 void switch_cop(struct mm_struct *next)
 {
 #ifdef CONFIG_PPC_ICSWX_PID
 	mtspr(SPRN_PID, next->context.cop_pid);
 #endif
 	mtspr(SPRN_ACOP, next->context.acop);
 }

 /**
  * Start using a coprocessor.
  * @acop: mask of coprocessor to be used.
  * @mm: The mm the coprocessor to associate with. Most likely current mm.
  *
  * Return a positive PID if successful. Negative errno otherwise.
  * The returned PID will be fed to the coprocessor to determine if an
  * icswx transaction is authenticated.
  */
 int use_cop(unsigned long acop, struct mm_struct *mm)
 {
 	int ret;

 	if (!cpu_has_feature(CPU_FTR_ICSWX))
 		return -ENODEV;

 	if (!mm || !acop)
 		return -EINVAL;

 	/* The page_table_lock ensures mm_users won't change under us */
 	spin_lock(&mm->page_table_lock);
 	spin_lock(mm->context.cop_lockp);

 	ret = get_cop_pid(mm);
 	if (ret < 0)
 		goto out;

 	/* update acop */
 	mm->context.acop |= acop;

 	sync_cop(mm);

 	/*
 	 * If this is a threaded process then there might be other threads
 	 * running. We need to send an IPI to force them to pick up any
 	 * change in PID and ACOP.
 	 */
 	if (atomic_read(&mm->mm_users) > 1)
 		smp_call_function(sync_cop, mm, 1);

 out:
 	spin_unlock(mm->context.cop_lockp);
 	spin_unlock(&mm->page_table_lock);

 	return ret;
 }
 EXPORT_SYMBOL_GPL(use_cop);

 /**
  * Stop using a coprocessor.
  * @acop: mask of coprocessor to be stopped.
  * @mm: The mm the coprocessor associated with.
  */
 void drop_cop(unsigned long acop, struct mm_struct *mm)
 {
 	int free_pid;

 	if (!cpu_has_feature(CPU_FTR_ICSWX))
 		return;

 	if (WARN_ON_ONCE(!mm))
 		return;

 	/* The page_table_lock ensures mm_users won't change under us */
 	spin_lock(&mm->page_table_lock);
 	spin_lock(mm->context.cop_lockp);

 	mm->context.acop &= ~acop;

 	free_pid = disable_cop_pid(mm);
 	sync_cop(mm);

 	/*
 	 * If this is a threaded process then there might be other threads
 	 * running. We need to send an IPI to force them to pick up any
 	 * change in PID and ACOP.
 	 */
 	if (atomic_read(&mm->mm_users) > 1)
 		smp_call_function(sync_cop, mm, 1);

 	if (free_pid != COP_PID_NONE)
 		free_cop_pid(free_pid);

 	spin_unlock(mm->context.cop_lockp);
 	spin_unlock(&mm->page_table_lock);
 }
 EXPORT_SYMBOL_GPL(drop_cop);

 static int acop_use_cop(int ct)
 {
 	/* There is no alternate policy, yet */
 	return -1;
 }

 /*
  * Get the instruction word at the NIP
  */
 static u32 acop_get_inst(struct pt_regs *regs)
 {
 	u32 inst;
 	u32 __user *p;

 	p = (u32 __user *)regs->nip;
 	if (!access_ok(VERIFY_READ, p, sizeof(*p)))
 		return 0;

 	if (__get_user(inst, p))
 		return 0;

 	return inst;
 }

 /**
  * @regs: registers at time of interrupt
  * @address: storage address
  * @error_code: Fault code, usually the DSISR or ESR depending on
  *		processor type
  *
  * Return 0 if we are able to resolve the data storage fault that
  * results from a CT miss in the ACOP register.
  */
 int acop_handle_fault(struct pt_regs *regs, unsigned long address,
 		      unsigned long error_code)
 {
 	int ct;
 	u32 inst = 0;

 	if (!cpu_has_feature(CPU_FTR_ICSWX)) {
 		pr_info("No coprocessors available");
 		_exception(SIGILL, regs, ILL_ILLOPN, address);
 	}

 	if (!user_mode(regs)) {
 		/* this could happen if the HV denies the
 		 * kernel access, for now we just die */
 		die("ICSWX from kernel failed", regs, SIGSEGV);
 	}

 	/* Some implementations leave us a hint for the CT */
 	ct = ICSWX_GET_CT_HINT(error_code);
 	if (ct < 0) {
 		/* we have to peek at the instruction word to figure out CT */
 		u32 ccw;
 		u32 rs;

 		inst = acop_get_inst(regs);
 		if (inst == 0)
 			return -1;

 		rs = (inst >> (31 - 10)) & 0x1f;
 		ccw = regs->gpr[rs];
 		ct = (ccw >> 16) & 0x3f;
 	}

 	/*
 	 * We could be here because another thread has enabled acop
 	 * but the ACOP register has yet to be updated.
 	 *
 	 * This should have been taken care of by the IPI to sync all
 	 * the threads (see smp_call_function(sync_cop, mm, 1)), but
 	 * that could take forever if there are a significant amount
 	 * of threads.
 	 *
 	 * Given the number of threads on some of these systems,
 	 * perhaps this is the best way to sync ACOP rather than whack
 	 * every thread with an IPI.
 	 */
 	if ((acop_copro_type_bit(ct) & current->active_mm->context.acop) != 0) {
 		sync_cop(current->active_mm);
 		return 0;
 	}

 	/* check for alternate policy */
 	if (!acop_use_cop(ct))
 		return 0;

 	/* at this point the CT is unknown to the system */
 	pr_warn("%s[%d]: Coprocessor %d is unavailable\n",
 		current->comm, current->pid, ct);

 	/* get inst if we don't already have it */
 	if (inst == 0) {
 		inst = acop_get_inst(regs);
 		if (inst == 0)
 			return -1;
 	}

 	/* Check if the instruction is the "record form" */
 	if (inst & 1) {
 		/*
 		 * the instruction is "record" form so we can reject
 		 * using CR0
 		 */
 		regs->ccr &= ~(0xful << 28);
 		regs->ccr |= ICSWX_RC_NOT_FOUND << 28;

 		/* Move on to the next instruction */
 		regs->nip += 4;
 	} else {
 		/*
 		 * There is no architected mechanism to report a bad
 		 * CT so we could either SIGILL or report nothing.
 		 * Since the non-record version should only bu used
 		 * for "hints" or "don't care" we should probably do
 		 * nothing.  However, I could see how some people
 		 * might want an SIGILL so it here if you want it.
 		 */
 #ifdef CONFIG_PPC_ICSWX_USE_SIGILL
 		_exception(SIGILL, regs, ILL_ILLOPN, address);
 #else
 		regs->nip += 4;
 #endif
 	}

 	return 0;
 }
 EXPORT_SYMBOL_GPL(acop_handle_fault);
	/*
	* ICSWX and ACOP Management
	*
	* Copyright (C) 2011 Anton Blanchard, IBM Corp. <anton@samba.org>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public License
	* as published by the Free Software Foundation; either version
	* 2 of the License, or (at your option) any later version.
	*
	*/

	#include <linux/sched.h>
	#include <linux/kernel.h>
	#include <linux/errno.h>
	#include <linux/types.h>
	#include <linux/mm.h>
	#include <linux/spinlock.h>
	#include <linux/module.h>
	#include <linux/uaccess.h>

	#include "icswx.h"

	/*
	* The processor and its L2 cache cause the icswx instruction to
	* generate a COP_REQ transaction on PowerBus. The transaction has no
	* address, and the processor does not perform an MMU access to
	* authenticate the transaction. The command portion of the PowerBus
	* COP_REQ transaction includes the LPAR_ID (LPID) and the coprocessor
	* Process ID (PID), which the coprocessor compares to the authorized
	* LPID and PID held in the coprocessor, to determine if the process
	* is authorized to generate the transaction. The data of the COP_REQ
	* transaction is 128-byte or less in size and is placed in cacheable
	* memory on a 128-byte cache line boundary.
	*
	* The task to use a coprocessor should use use_cop() to mark the use
	* of the Coprocessor Type (CT) and context switching. On a server
	* class processor, the PID register is used only for coprocessor
	* management + * and so a coprocessor PID is allocated before
	* executing icswx + * instruction. Drop_cop() is used to free the
	* coprocessor PID.
	*
	* Example:
	* Host Fabric Interface (HFI) is a PowerPC network coprocessor.
	* Each HFI have multiple windows. Each HFI window serves as a
	* network device sending to and receiving from HFI network.
	* HFI immediate send function uses icswx instruction. The immediate
	* send function allows small (single cache-line) packets be sent
	* without using the regular HFI send FIFO and doorbell, which are
	* much slower than immediate send.
	*
	* For each task intending to use HFI immediate send, the HFI driver
	* calls use_cop() to obtain a coprocessor PID for the task.
	* The HFI driver then allocate a free HFI window and save the
	* coprocessor PID to the HFI window to allow the task to use the
	* HFI window.
	*
	* The HFI driver repeatedly creates immediate send packets and
	* issues icswx instruction to send data through the HFI window.
	* The HFI compares the coprocessor PID in the CPU PID register
	* to the PID held in the HFI window to determine if the transaction
	* is allowed.
	*
	* When the task to release the HFI window, the HFI driver calls
	* drop_cop() to release the coprocessor PID.
	*/

	void switch_cop(struct mm_struct *next)
	{
	#ifdef CONFIG_PPC_ICSWX_PID
	mtspr(SPRN_PID, next->context.cop_pid);
	#endif
	mtspr(SPRN_ACOP, next->context.acop);
	}

	/**
	* Start using a coprocessor.
	* @acop: mask of coprocessor to be used.
	* @mm: The mm the coprocessor to associate with. Most likely current mm.
	*
	* Return a positive PID if successful. Negative errno otherwise.
	* The returned PID will be fed to the coprocessor to determine if an
	* icswx transaction is authenticated.
	*/
	int use_cop(unsigned long acop, struct mm_struct *mm)
	{
	int ret;

	if (!cpu_has_feature(CPU_FTR_ICSWX))
	return -ENODEV;

	if (!mm \|\| !acop)
	return -EINVAL;

	/* The page_table_lock ensures mm_users won't change under us */
	spin_lock(&mm->page_table_lock);
	spin_lock(mm->context.cop_lockp);

	ret = get_cop_pid(mm);
	if (ret < 0)
	goto out;

	/* update acop */
	mm->context.acop \|= acop;

	sync_cop(mm);

	/*
	* If this is a threaded process then there might be other threads
	* running. We need to send an IPI to force them to pick up any
	* change in PID and ACOP.
	*/
	if (atomic_read(&mm->mm_users) > 1)
	smp_call_function(sync_cop, mm, 1);

	out:
	spin_unlock(mm->context.cop_lockp);
	spin_unlock(&mm->page_table_lock);

	return ret;
	}
	EXPORT_SYMBOL_GPL(use_cop);

	/**
	* Stop using a coprocessor.
	* @acop: mask of coprocessor to be stopped.
	* @mm: The mm the coprocessor associated with.
	*/
	void drop_cop(unsigned long acop, struct mm_struct *mm)
	{
	int free_pid;

	if (!cpu_has_feature(CPU_FTR_ICSWX))
	return;

	if (WARN_ON_ONCE(!mm))
	return;

	/* The page_table_lock ensures mm_users won't change under us */
	spin_lock(&mm->page_table_lock);
	spin_lock(mm->context.cop_lockp);

	mm->context.acop &= ~acop;

	free_pid = disable_cop_pid(mm);
	sync_cop(mm);

	/*
	* If this is a threaded process then there might be other threads
	* running. We need to send an IPI to force them to pick up any
	* change in PID and ACOP.
	*/
	if (atomic_read(&mm->mm_users) > 1)
	smp_call_function(sync_cop, mm, 1);

	if (free_pid != COP_PID_NONE)
	free_cop_pid(free_pid);

	spin_unlock(mm->context.cop_lockp);
	spin_unlock(&mm->page_table_lock);
	}
	EXPORT_SYMBOL_GPL(drop_cop);

	static int acop_use_cop(int ct)
	{
	/* There is no alternate policy, yet */
	return -1;
	}

	/*
	* Get the instruction word at the NIP
	*/
	static u32 acop_get_inst(struct pt_regs *regs)
	{
	u32 inst;
	u32 __user *p;

	p = (u32 __user *)regs->nip;
	if (!access_ok(VERIFY_READ, p, sizeof(*p)))
	return 0;

	if (__get_user(inst, p))
	return 0;

	return inst;
	}

	/**
	* @regs: registers at time of interrupt
	* @address: storage address
	* @error_code: Fault code, usually the DSISR or ESR depending on
	* processor type
	*
	* Return 0 if we are able to resolve the data storage fault that
	* results from a CT miss in the ACOP register.
	*/
	int acop_handle_fault(struct pt_regs *regs, unsigned long address,
	unsigned long error_code)
	{
	int ct;
	u32 inst = 0;

	if (!cpu_has_feature(CPU_FTR_ICSWX)) {
	pr_info("No coprocessors available");
	_exception(SIGILL, regs, ILL_ILLOPN, address);
	}

	if (!user_mode(regs)) {
	/* this could happen if the HV denies the
	* kernel access, for now we just die */
	die("ICSWX from kernel failed", regs, SIGSEGV);
	}

	/* Some implementations leave us a hint for the CT */
	ct = ICSWX_GET_CT_HINT(error_code);
	if (ct < 0) {
	/* we have to peek at the instruction word to figure out CT */
	u32 ccw;
	u32 rs;

	inst = acop_get_inst(regs);
	if (inst == 0)
	return -1;

	rs = (inst >> (31 - 10)) & 0x1f;
	ccw = regs->gpr[rs];
	ct = (ccw >> 16) & 0x3f;
	}

	/*
	* We could be here because another thread has enabled acop
	* but the ACOP register has yet to be updated.
	*
	* This should have been taken care of by the IPI to sync all
	* the threads (see smp_call_function(sync_cop, mm, 1)), but
	* that could take forever if there are a significant amount
	* of threads.
	*
	* Given the number of threads on some of these systems,
	* perhaps this is the best way to sync ACOP rather than whack
	* every thread with an IPI.
	*/
	if ((acop_copro_type_bit(ct) & current->active_mm->context.acop) != 0) {
	sync_cop(current->active_mm);
	return 0;
	}

	/* check for alternate policy */
	if (!acop_use_cop(ct))
	return 0;

	/* at this point the CT is unknown to the system */
	pr_warn("%s[%d]: Coprocessor %d is unavailable\n",
	current->comm, current->pid, ct);

	/* get inst if we don't already have it */
	if (inst == 0) {
	inst = acop_get_inst(regs);
	if (inst == 0)
	return -1;
	}

	/* Check if the instruction is the "record form" */
	if (inst & 1) {
	/*
	* the instruction is "record" form so we can reject
	* using CR0
	*/
	regs->ccr &= ~(0xful << 28);
	regs->ccr \|= ICSWX_RC_NOT_FOUND << 28;

	/* Move on to the next instruction */
	regs->nip += 4;
	} else {
	/*
	* There is no architected mechanism to report a bad
	* CT so we could either SIGILL or report nothing.
	* Since the non-record version should only bu used
	* for "hints" or "don't care" we should probably do
	* nothing. However, I could see how some people
	* might want an SIGILL so it here if you want it.
	*/
	#ifdef CONFIG_PPC_ICSWX_USE_SIGILL
	_exception(SIGILL, regs, ILL_ILLOPN, address);
	#else
	regs->nip += 4;
	#endif
	}

	return 0;
	}
	EXPORT_SYMBOL_GPL(acop_handle_fault);