xref: /arch/metag/lib/usercopy.c

/*
 * User address space access functions.
 * The non-inlined parts of asm-metag/uaccess.h are here.
 *
 * Copyright (C) 2006, Imagination Technologies.
 * Copyright (C) 2000, Axis Communications AB.
 *
 * Written by Hans-Peter Nilsson.
 * Pieces used from memcpy, originally by Kenny Ranerup long time ago.
 * Modified for Meta by Will Newton.
 */

#include <linux/export.h>
#include <linux/uaccess.h>
#include <asm/cache.h>			/* def of L1_CACHE_BYTES */

#define USE_RAPF
#define RAPF_MIN_BUF_SIZE	(3*L1_CACHE_BYTES)


/* The "double write" in this code is because the Meta will not fault
 * immediately unless the memory pipe is forced to by e.g. a data stall or
 * another memory op. The second write should be discarded by the write
 * combiner so should have virtually no cost.
 */

#define __asm_copy_user_cont(to, from, ret, COPY, FIXUP, TENTRY) \
	asm volatile (						 \
		COPY						 \
		"1:\n"						 \
		"	.section .fixup,\"ax\"\n"		 \
		"	MOV D1Ar1,#0\n"				 \
		FIXUP						 \
		"	MOVT    D1Ar1,#HI(1b)\n"		 \
		"	JUMP    D1Ar1,#LO(1b)\n"		 \
		"	.previous\n"				 \
		"	.section __ex_table,\"a\"\n"		 \
		TENTRY						 \
		"	.previous\n"				 \
		: "=r" (to), "=r" (from), "=r" (ret)		 \
		: "0" (to), "1" (from), "2" (ret)		 \
		: "D1Ar1", "memory")


#define __asm_copy_to_user_1(to, from, ret)	\
	__asm_copy_user_cont(to, from, ret,	\
		"	GETB D1Ar1,[%1++]\n"	\
		"	SETB [%0],D1Ar1\n"	\
		"2:	SETB [%0++],D1Ar1\n",	\
		"3:	ADD  %2,%2,#1\n",	\
		"	.long 2b,3b\n")

#define __asm_copy_to_user_2x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
	__asm_copy_user_cont(to, from, ret,		\
		"	GETW D1Ar1,[%1++]\n"		\
		"	SETW [%0],D1Ar1\n"		\
		"2:	SETW [%0++],D1Ar1\n" COPY,	\
		"3:	ADD  %2,%2,#2\n" FIXUP,		\
		"	.long 2b,3b\n" TENTRY)

#define __asm_copy_to_user_2(to, from, ret) \
	__asm_copy_to_user_2x_cont(to, from, ret, "", "", "")

#define __asm_copy_to_user_3(to, from, ret) \
	__asm_copy_to_user_2x_cont(to, from, ret,	\
		"	GETB D1Ar1,[%1++]\n"		\
		"	SETB [%0],D1Ar1\n"		\
		"4:	SETB [%0++],D1Ar1\n",		\
		"5:	ADD  %2,%2,#1\n",		\
		"	.long 4b,5b\n")

#define __asm_copy_to_user_4x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
	__asm_copy_user_cont(to, from, ret,		\
		"	GETD D1Ar1,[%1++]\n"		\
		"	SETD [%0],D1Ar1\n"		\
		"2:	SETD [%0++],D1Ar1\n" COPY,	\
		"3:	ADD  %2,%2,#4\n" FIXUP,		\
		"	.long 2b,3b\n" TENTRY)

#define __asm_copy_to_user_4(to, from, ret) \
	__asm_copy_to_user_4x_cont(to, from, ret, "", "", "")

#define __asm_copy_to_user_5(to, from, ret) \
	__asm_copy_to_user_4x_cont(to, from, ret,	\
		"	GETB D1Ar1,[%1++]\n"		\
		"	SETB [%0],D1Ar1\n"		\
		"4:	SETB [%0++],D1Ar1\n",		\
		"5:	ADD  %2,%2,#1\n",		\
		"	.long 4b,5b\n")

#define __asm_copy_to_user_6x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
	__asm_copy_to_user_4x_cont(to, from, ret,	\
		"	GETW D1Ar1,[%1++]\n"		\
		"	SETW [%0],D1Ar1\n"		\
		"4:	SETW [%0++],D1Ar1\n" COPY,	\
		"5:	ADD  %2,%2,#2\n" FIXUP,		\
		"	.long 4b,5b\n" TENTRY)

#define __asm_copy_to_user_6(to, from, ret) \
	__asm_copy_to_user_6x_cont(to, from, ret, "", "", "")

#define __asm_copy_to_user_7(to, from, ret) \
	__asm_copy_to_user_6x_cont(to, from, ret,	\
		"	GETB D1Ar1,[%1++]\n"		\
		"	SETB [%0],D1Ar1\n"		\
		"6:	SETB [%0++],D1Ar1\n",		\
		"7:	ADD  %2,%2,#1\n",		\
		"	.long 6b,7b\n")

#define __asm_copy_to_user_8x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
	__asm_copy_to_user_4x_cont(to, from, ret,	\
		"	GETD D1Ar1,[%1++]\n"		\
		"	SETD [%0],D1Ar1\n"		\
		"4:	SETD [%0++],D1Ar1\n" COPY,	\
		"5:	ADD  %2,%2,#4\n"  FIXUP,	\
		"	.long 4b,5b\n" TENTRY)

#define __asm_copy_to_user_8(to, from, ret) \
	__asm_copy_to_user_8x_cont(to, from, ret, "", "", "")

#define __asm_copy_to_user_9(to, from, ret) \
	__asm_copy_to_user_8x_cont(to, from, ret,	\
		"	GETB D1Ar1,[%1++]\n"		\
		"	SETB [%0],D1Ar1\n"		\
		"6:	SETB [%0++],D1Ar1\n",		\
		"7:	ADD  %2,%2,#1\n",		\
		"	.long 6b,7b\n")

#define __asm_copy_to_user_10x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
	__asm_copy_to_user_8x_cont(to, from, ret,	\
		"	GETW D1Ar1,[%1++]\n"		\
		"	SETW [%0],D1Ar1\n"		\
		"6:	SETW [%0++],D1Ar1\n" COPY,	\
		"7:	ADD  %2,%2,#2\n" FIXUP,		\
		"	.long 6b,7b\n" TENTRY)

#define __asm_copy_to_user_10(to, from, ret) \
	__asm_copy_to_user_10x_cont(to, from, ret, "", "", "")

#define __asm_copy_to_user_11(to, from, ret) \
	__asm_copy_to_user_10x_cont(to, from, ret,	\
		"	GETB D1Ar1,[%1++]\n"		\
		"	SETB [%0],D1Ar1\n"		\
		"8:	SETB [%0++],D1Ar1\n",		\
		"9:	ADD  %2,%2,#1\n",		\
		"	.long 8b,9b\n")

#define __asm_copy_to_user_12x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
	__asm_copy_to_user_8x_cont(to, from, ret,	\
		"	GETD D1Ar1,[%1++]\n"		\
		"	SETD [%0],D1Ar1\n"		\
		"6:	SETD [%0++],D1Ar1\n" COPY,	\
		"7:	ADD  %2,%2,#4\n" FIXUP,		\
		"	.long 6b,7b\n" TENTRY)
#define __asm_copy_to_user_12(to, from, ret) \
	__asm_copy_to_user_12x_cont(to, from, ret, "", "", "")

#define __asm_copy_to_user_13(to, from, ret) \
	__asm_copy_to_user_12x_cont(to, from, ret,	\
		"	GETB D1Ar1,[%1++]\n"		\
		"	SETB [%0],D1Ar1\n"		\
		"8:	SETB [%0++],D1Ar1\n",		\
		"9:	ADD  %2,%2,#1\n",		\
		"	.long 8b,9b\n")

#define __asm_copy_to_user_14x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
	__asm_copy_to_user_12x_cont(to, from, ret,	\
		"	GETW D1Ar1,[%1++]\n"		\
		"	SETW [%0],D1Ar1\n"		\
		"8:	SETW [%0++],D1Ar1\n" COPY,	\
		"9:	ADD  %2,%2,#2\n" FIXUP,		\
		"	.long 8b,9b\n" TENTRY)

#define __asm_copy_to_user_14(to, from, ret) \
	__asm_copy_to_user_14x_cont(to, from, ret, "", "", "")

#define __asm_copy_to_user_15(to, from, ret) \
	__asm_copy_to_user_14x_cont(to, from, ret,	\
		"	GETB D1Ar1,[%1++]\n"		\
		"	SETB [%0],D1Ar1\n"		\
		"10:	SETB [%0++],D1Ar1\n",		\
		"11:	ADD  %2,%2,#1\n",		\
		"	.long 10b,11b\n")

#define __asm_copy_to_user_16x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
	__asm_copy_to_user_12x_cont(to, from, ret,	\
		"	GETD D1Ar1,[%1++]\n"		\
		"	SETD [%0],D1Ar1\n"		\
		"8:	SETD [%0++],D1Ar1\n" COPY,	\
		"9:	ADD  %2,%2,#4\n" FIXUP,		\
		"	.long 8b,9b\n" TENTRY)

#define __asm_copy_to_user_16(to, from, ret) \
		__asm_copy_to_user_16x_cont(to, from, ret, "", "", "")

#define __asm_copy_to_user_8x64(to, from, ret) \
	asm volatile (					\
		"	GETL D0Ar2,D1Ar1,[%1++]\n"	\
		"	SETL [%0],D0Ar2,D1Ar1\n"	\
		"2:	SETL [%0++],D0Ar2,D1Ar1\n"	\
		"1:\n"					\
		"	.section .fixup,\"ax\"\n"	\
		"3:	ADD  %2,%2,#8\n"		\
		"	MOVT    D0Ar2,#HI(1b)\n"	\
		"	JUMP    D0Ar2,#LO(1b)\n"	\
		"	.previous\n"			\
		"	.section __ex_table,\"a\"\n"	\
		"	.long 2b,3b\n"			\
		"	.previous\n"			\
		: "=r" (to), "=r" (from), "=r" (ret)	\
		: "0" (to), "1" (from), "2" (ret)	\
		: "D1Ar1", "D0Ar2", "memory")

/*
 *	optimized copying loop using RAPF when 64 bit aligned
 *
 *	n		will be automatically decremented inside the loop
 *	ret		will be left intact. if error occurs we will rewind
 *			so that the original non optimized code will fill up
 *			this value correctly.
 *
 *	on fault:
 *		>	n will hold total number of uncopied bytes
 *
 *		>	{'to','from'} will be rewind back so that
 *			the non-optimized code will do the proper fix up
 *
 *	DCACHE drops the cacheline which helps in reducing cache
 *	pollution.
 *
 *	We introduce an extra SETL at the end of the loop to
 *	ensure we don't fall off the loop before we catch all
 *	erros.
 *
 *	NOTICE:
 *		LSM_STEP in TXSTATUS must be cleared in fix up code.
 *		since we're using M{S,G}ETL, a fault might happen at
 *		any address in the middle of M{S,G}ETL causing
 *		the value of LSM_STEP to be incorrect which can
 *		cause subsequent use of M{S,G}ET{L,D} to go wrong.
 *		ie: if LSM_STEP was 1 when a fault occurs, the
 *		next call to M{S,G}ET{L,D} will skip the first
 *		copy/getting as it think that the first 1 has already
 *		been done.
 *
 */
#define __asm_copy_user_64bit_rapf_loop(				\
		to, from, ret, n, id, FIXUP)				\
	asm volatile (							\
		".balign 8\n"						\
		"MOV	RAPF, %1\n"					\
		"MSETL	[A0StP++], D0Ar6, D0FrT, D0.5, D0.6, D0.7\n"	\
		"MOV	D0Ar6, #0\n"					\
		"LSR	D1Ar5, %3, #6\n"				\
		"SUB	TXRPT, D1Ar5, #2\n"				\
		"MOV	RAPF, %1\n"					\
		"$Lloop"id":\n"						\
		"ADD	RAPF, %1, #64\n"				\
		"21:\n"							\
		"MGETL	D0FrT, D0.5, D0.6, D0.7, [%1++]\n"		\
		"22:\n"							\
		"MSETL	[%0++], D0FrT, D0.5, D0.6, D0.7\n"		\
		"SUB	%3, %3, #32\n"					\
		"23:\n"							\
		"MGETL	D0FrT, D0.5, D0.6, D0.7, [%1++]\n"		\
		"24:\n"							\
		"MSETL	[%0++], D0FrT, D0.5, D0.6, D0.7\n"		\
		"SUB	%3, %3, #32\n"					\
		"DCACHE	[%1+#-64], D0Ar6\n"				\
		"BR	$Lloop"id"\n"					\
									\
		"MOV	RAPF, %1\n"					\
		"25:\n"							\
		"MGETL	D0FrT, D0.5, D0.6, D0.7, [%1++]\n"		\
		"26:\n"							\
		"MSETL	[%0++], D0FrT, D0.5, D0.6, D0.7\n"		\
		"SUB	%3, %3, #32\n"					\
		"27:\n"							\
		"MGETL	D0FrT, D0.5, D0.6, D0.7, [%1++]\n"		\
		"28:\n"							\
		"MSETL	[%0++], D0FrT, D0.5, D0.6, D0.7\n"		\
		"SUB	%0, %0, #8\n"					\
		"29:\n"							\
		"SETL	[%0++], D0.7, D1.7\n"				\
		"SUB	%3, %3, #32\n"					\
		"1:"							\
		"DCACHE	[%1+#-64], D0Ar6\n"				\
		"GETL    D0Ar6, D1Ar5, [A0StP+#-40]\n"			\
		"GETL    D0FrT, D1RtP, [A0StP+#-32]\n"			\
		"GETL    D0.5, D1.5, [A0StP+#-24]\n"			\
		"GETL    D0.6, D1.6, [A0StP+#-16]\n"			\
		"GETL    D0.7, D1.7, [A0StP+#-8]\n"			\
		"SUB A0StP, A0StP, #40\n"				\
		"	.section .fixup,\"ax\"\n"			\
		"4:\n"							\
		"	ADD	%0, %0, #8\n"				\
		"3:\n"							\
		"	MOV	D0Ar2, TXSTATUS\n"			\
		"	MOV	D1Ar1, TXSTATUS\n"			\
		"	AND	D1Ar1, D1Ar1, #0xFFFFF8FF\n"		\
		"	MOV	TXSTATUS, D1Ar1\n"			\
			FIXUP						\
		"	MOVT    D0Ar2,#HI(1b)\n"			\
		"	JUMP    D0Ar2,#LO(1b)\n"			\
		"	.previous\n"					\
		"	.section __ex_table,\"a\"\n"			\
		"	.long 21b,3b\n"					\
		"	.long 22b,3b\n"					\
		"	.long 23b,3b\n"					\
		"	.long 24b,3b\n"					\
		"	.long 25b,3b\n"					\
		"	.long 26b,3b\n"					\
		"	.long 27b,3b\n"					\
		"	.long 28b,3b\n"					\
		"	.long 29b,4b\n"					\
		"	.previous\n"					\
		: "=r" (to), "=r" (from), "=r" (ret), "=d" (n)		\
		: "0" (to), "1" (from), "2" (ret), "3" (n)		\
		: "D1Ar1", "D0Ar2", "memory")

/*	rewind 'to' and 'from'  pointers when a fault occurs
 *
 *	Rationale:
 *		A fault always occurs on writing to user buffer. A fault
 *		is at a single address, so we need to rewind by only 4
 *		bytes.
 *		Since we do a complete read from kernel buffer before
 *		writing, we need to rewind it also. The amount to be
 *		rewind equals the number of faulty writes in MSETD
 *		which is: [4 - (LSM_STEP-1)]*8
 *		LSM_STEP is bits 10:8 in TXSTATUS which is already read
 *		and stored in D0Ar2
 *
 *		NOTE: If a fault occurs at the last operation in M{G,S}ETL
 *			LSM_STEP will be 0. ie: we do 4 writes in our case, if
 *			a fault happens at the 4th write, LSM_STEP will be 0
 *			instead of 4. The code copes with that.
 *
 *		n is updated by the number of successful writes, which is:
 *		n = n - (LSM_STEP-1)*8
 */
#define __asm_copy_to_user_64bit_rapf_loop(to,	from, ret, n, id)\
	__asm_copy_user_64bit_rapf_loop(to, from, ret, n, id,		\
		"LSR	D0Ar2, D0Ar2, #8\n"				\
		"AND	D0Ar2, D0Ar2, #0x7\n"				\
		"ADDZ	D0Ar2, D0Ar2, #4\n"				\
		"SUB	D0Ar2, D0Ar2, #1\n"				\
		"MOV	D1Ar1, #4\n"					\
		"SUB	D0Ar2, D1Ar1, D0Ar2\n"				\
		"LSL	D0Ar2, D0Ar2, #3\n"				\
		"LSL	D1Ar1, D1Ar1, #3\n"				\
		"SUB	D1Ar1, D1Ar1, D0Ar2\n"				\
		"SUB	%0, %0, #8\n"					\
		"SUB	%1,	%1,D0Ar2\n"				\
		"SUB	%3, %3, D1Ar1\n")

/*
 *	optimized copying loop using RAPF when 32 bit aligned
 *
 *	n		will be automatically decremented inside the loop
 *	ret		will be left intact. if error occurs we will rewind
 *			so that the original non optimized code will fill up
 *			this value correctly.
 *
 *	on fault:
 *		>	n will hold total number of uncopied bytes
 *
 *		>	{'to','from'} will be rewind back so that
 *			the non-optimized code will do the proper fix up
 *
 *	DCACHE drops the cacheline which helps in reducing cache
 *	pollution.
 *
 *	We introduce an extra SETD at the end of the loop to
 *	ensure we don't fall off the loop before we catch all
 *	erros.
 *
 *	NOTICE:
 *		LSM_STEP in TXSTATUS must be cleared in fix up code.
 *		since we're using M{S,G}ETL, a fault might happen at
 *		any address in the middle of M{S,G}ETL causing
 *		the value of LSM_STEP to be incorrect which can
 *		cause subsequent use of M{S,G}ET{L,D} to go wrong.
 *		ie: if LSM_STEP was 1 when a fault occurs, the
 *		next call to M{S,G}ET{L,D} will skip the first
 *		copy/getting as it think that the first 1 has already
 *		been done.
 *
 */
#define __asm_copy_user_32bit_rapf_loop(				\
			to,	from, ret, n, id, FIXUP)		\
	asm volatile (							\
		".balign 8\n"						\
		"MOV	RAPF, %1\n"					\
		"MSETL	[A0StP++], D0Ar6, D0FrT, D0.5, D0.6, D0.7\n"	\
		"MOV	D0Ar6, #0\n"					\
		"LSR	D1Ar5, %3, #6\n"				\
		"SUB	TXRPT, D1Ar5, #2\n"				\
		"MOV	RAPF, %1\n"					\
	"$Lloop"id":\n"							\
		"ADD	RAPF, %1, #64\n"				\
		"21:\n"							\
		"MGETD	D0FrT, D0.5, D0.6, D0.7, [%1++]\n"		\
		"22:\n"							\
		"MSETD	[%0++], D0FrT, D0.5, D0.6, D0.7\n"		\
		"SUB	%3, %3, #16\n"					\
		"23:\n"							\
		"MGETD	D0FrT, D0.5, D0.6, D0.7, [%1++]\n"		\
		"24:\n"							\
		"MSETD	[%0++], D0FrT, D0.5, D0.6, D0.7\n"		\
		"SUB	%3, %3, #16\n"					\
		"25:\n"							\
		"MGETD	D0FrT, D0.5, D0.6, D0.7, [%1++]\n"		\
		"26:\n"							\
		"MSETD	[%0++], D0FrT, D0.5, D0.6, D0.7\n"		\
		"SUB	%3, %3, #16\n"					\
		"27:\n"							\
		"MGETD	D0FrT, D0.5, D0.6, D0.7, [%1++]\n"		\
		"28:\n"							\
		"MSETD	[%0++], D0FrT, D0.5, D0.6, D0.7\n"		\
		"SUB	%3, %3, #16\n"					\
		"DCACHE	[%1+#-64], D0Ar6\n"				\
		"BR	$Lloop"id"\n"					\
									\
		"MOV	RAPF, %1\n"					\
		"29:\n"							\
		"MGETD	D0FrT, D0.5, D0.6, D0.7, [%1++]\n"		\
		"30:\n"							\
		"MSETD	[%0++], D0FrT, D0.5, D0.6, D0.7\n"		\
		"SUB	%3, %3, #16\n"					\
		"31:\n"							\
		"MGETD	D0FrT, D0.5, D0.6, D0.7, [%1++]\n"		\
		"32:\n"							\
		"MSETD	[%0++], D0FrT, D0.5, D0.6, D0.7\n"		\
		"SUB	%3, %3, #16\n"					\
		"33:\n"							\
		"MGETD	D0FrT, D0.5, D0.6, D0.7, [%1++]\n"		\
		"34:\n"							\
		"MSETD	[%0++], D0FrT, D0.5, D0.6, D0.7\n"		\
		"SUB	%3, %3, #16\n"					\
		"35:\n"							\
		"MGETD	D0FrT, D0.5, D0.6, D0.7, [%1++]\n"		\
		"36:\n"							\
		"MSETD	[%0++], D0FrT, D0.5, D0.6, D0.7\n"		\
		"SUB	%0, %0, #4\n"					\
		"37:\n"							\
		"SETD	[%0++], D0.7\n"					\
		"SUB	%3, %3, #16\n"					\
		"1:"							\
		"DCACHE	[%1+#-64], D0Ar6\n"				\
		"GETL    D0Ar6, D1Ar5, [A0StP+#-40]\n"			\
		"GETL    D0FrT, D1RtP, [A0StP+#-32]\n"			\
		"GETL    D0.5, D1.5, [A0StP+#-24]\n"			\
		"GETL    D0.6, D1.6, [A0StP+#-16]\n"			\
		"GETL    D0.7, D1.7, [A0StP+#-8]\n"			\
		"SUB A0StP, A0StP, #40\n"				\
		"	.section .fixup,\"ax\"\n"			\
		"4:\n"							\
		"	ADD		%0, %0, #4\n"			\
		"3:\n"							\
		"	MOV	D0Ar2, TXSTATUS\n"			\
		"	MOV	D1Ar1, TXSTATUS\n"			\
		"	AND	D1Ar1, D1Ar1, #0xFFFFF8FF\n"		\
		"	MOV	TXSTATUS, D1Ar1\n"			\
			FIXUP						\
		"	MOVT    D0Ar2,#HI(1b)\n"			\
		"	JUMP    D0Ar2,#LO(1b)\n"			\
		"	.previous\n"					\
		"	.section __ex_table,\"a\"\n"			\
		"	.long 21b,3b\n"					\
		"	.long 22b,3b\n"					\
		"	.long 23b,3b\n"					\
		"	.long 24b,3b\n"					\
		"	.long 25b,3b\n"					\
		"	.long 26b,3b\n"					\
		"	.long 27b,3b\n"					\
		"	.long 28b,3b\n"					\
		"	.long 29b,3b\n"					\
		"	.long 30b,3b\n"					\
		"	.long 31b,3b\n"					\
		"	.long 32b,3b\n"					\
		"	.long 33b,3b\n"					\
		"	.long 34b,3b\n"					\
		"	.long 35b,3b\n"					\
		"	.long 36b,3b\n"					\
		"	.long 37b,4b\n"					\
		"	.previous\n"					\
		: "=r" (to), "=r" (from), "=r" (ret), "=d" (n)		\
		: "0" (to), "1" (from), "2" (ret), "3" (n)		\
		: "D1Ar1", "D0Ar2", "memory")

/*	rewind 'to' and 'from'  pointers when a fault occurs
 *
 *	Rationale:
 *		A fault always occurs on writing to user buffer. A fault
 *		is at a single address, so we need to rewind by only 4
 *		bytes.
 *		Since we do a complete read from kernel buffer before
 *		writing, we need to rewind it also. The amount to be
 *		rewind equals the number of faulty writes in MSETD
 *		which is: [4 - (LSM_STEP-1)]*4
 *		LSM_STEP is bits 10:8 in TXSTATUS which is already read
 *		and stored in D0Ar2
 *
 *		NOTE: If a fault occurs at the last operation in M{G,S}ETL
 *			LSM_STEP will be 0. ie: we do 4 writes in our case, if
 *			a fault happens at the 4th write, LSM_STEP will be 0
 *			instead of 4. The code copes with that.
 *
 *		n is updated by the number of successful writes, which is:
 *		n = n - (LSM_STEP-1)*4
 */
#define __asm_copy_to_user_32bit_rapf_loop(to, from, ret, n, id)\
	__asm_copy_user_32bit_rapf_loop(to, from, ret, n, id,		\
		"LSR	D0Ar2, D0Ar2, #8\n"				\
		"AND	D0Ar2, D0Ar2, #0x7\n"				\
		"ADDZ	D0Ar2, D0Ar2, #4\n"				\
		"SUB	D0Ar2, D0Ar2, #1\n"				\
		"MOV	D1Ar1, #4\n"					\
		"SUB	D0Ar2, D1Ar1, D0Ar2\n"				\
		"LSL	D0Ar2, D0Ar2, #2\n"				\
		"LSL	D1Ar1, D1Ar1, #2\n"				\
		"SUB	D1Ar1, D1Ar1, D0Ar2\n"				\
		"SUB	%0, %0, #4\n"					\
		"SUB	%1,	%1,	D0Ar2\n"			\
		"SUB	%3, %3, D1Ar1\n")

unsigned long __copy_user(void __user *pdst, const void *psrc,
			  unsigned long n)
{
	register char __user *dst asm ("A0.2") = pdst;
	register const char *src asm ("A1.2") = psrc;
	unsigned long retn = 0;

	if (n == 0)
		return 0;

	if ((unsigned long) src & 1) {
		__asm_copy_to_user_1(dst, src, retn);
		n--;
	}
	if ((unsigned long) dst & 1) {
		/* Worst case - byte copy */
		while (n > 0) {
			__asm_copy_to_user_1(dst, src, retn);
			n--;
		}
	}
	if (((unsigned long) src & 2) && n >= 2) {
		__asm_copy_to_user_2(dst, src, retn);
		n -= 2;
	}
	if ((unsigned long) dst & 2) {
		/* Second worst case - word copy */
		while (n >= 2) {
			__asm_copy_to_user_2(dst, src, retn);
			n -= 2;
		}
	}

#ifdef USE_RAPF
	/* 64 bit copy loop */
	if (!(((unsigned long) src | (__force unsigned long) dst) & 7)) {
		if (n >= RAPF_MIN_BUF_SIZE) {
			/* copy user using 64 bit rapf copy */
			__asm_copy_to_user_64bit_rapf_loop(dst, src, retn,
							n, "64cu");
		}
		while (n >= 8) {
			__asm_copy_to_user_8x64(dst, src, retn);
			n -= 8;
		}
	}
	if (n >= RAPF_MIN_BUF_SIZE) {
		/* copy user using 32 bit rapf copy */
		__asm_copy_to_user_32bit_rapf_loop(dst, src, retn, n, "32cu");
	}
#else
	/* 64 bit copy loop */
	if (!(((unsigned long) src | (__force unsigned long) dst) & 7)) {
		while (n >= 8) {
			__asm_copy_to_user_8x64(dst, src, retn);
			n -= 8;
		}
	}
#endif

	while (n >= 16) {
		__asm_copy_to_user_16(dst, src, retn);
		n -= 16;
	}

	while (n >= 4) {
		__asm_copy_to_user_4(dst, src, retn);
		n -= 4;
	}

	switch (n) {
	case 0:
		break;
	case 1:
		__asm_copy_to_user_1(dst, src, retn);
		break;
	case 2:
		__asm_copy_to_user_2(dst, src, retn);
		break;
	case 3:
		__asm_copy_to_user_3(dst, src, retn);
		break;
	}

	return retn;
}
EXPORT_SYMBOL(__copy_user);

#define __asm_copy_from_user_1(to, from, ret) \
	__asm_copy_user_cont(to, from, ret,	\
		"	GETB D1Ar1,[%1++]\n"	\
		"2:	SETB [%0++],D1Ar1\n",	\
		"3:	ADD  %2,%2,#1\n"	\
		"	SETB [%0++],D1Ar1\n",	\
		"	.long 2b,3b\n")

#define __asm_copy_from_user_2x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
	__asm_copy_user_cont(to, from, ret,		\
		"	GETW D1Ar1,[%1++]\n"		\
		"2:	SETW [%0++],D1Ar1\n" COPY,	\
		"3:	ADD  %2,%2,#2\n"		\
		"	SETW [%0++],D1Ar1\n" FIXUP,	\
		"	.long 2b,3b\n" TENTRY)

#define __asm_copy_from_user_2(to, from, ret) \
	__asm_copy_from_user_2x_cont(to, from, ret, "", "", "")

#define __asm_copy_from_user_3(to, from, ret)		\
	__asm_copy_from_user_2x_cont(to, from, ret,	\
		"	GETB D1Ar1,[%1++]\n"		\
		"4:	SETB [%0++],D1Ar1\n",		\
		"5:	ADD  %2,%2,#1\n"		\
		"	SETB [%0++],D1Ar1\n",		\
		"	.long 4b,5b\n")

#define __asm_copy_from_user_4x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
	__asm_copy_user_cont(to, from, ret,		\
		"	GETD D1Ar1,[%1++]\n"		\
		"2:	SETD [%0++],D1Ar1\n" COPY,	\
		"3:	ADD  %2,%2,#4\n"		\
		"	SETD [%0++],D1Ar1\n" FIXUP,	\
		"	.long 2b,3b\n" TENTRY)

#define __asm_copy_from_user_4(to, from, ret) \
	__asm_copy_from_user_4x_cont(to, from, ret, "", "", "")

#define __asm_copy_from_user_5(to, from, ret) \
	__asm_copy_from_user_4x_cont(to, from, ret,	\
		"	GETB D1Ar1,[%1++]\n"		\
		"4:	SETB [%0++],D1Ar1\n",		\
		"5:	ADD  %2,%2,#1\n"		\
		"	SETB [%0++],D1Ar1\n",		\
		"	.long 4b,5b\n")

#define __asm_copy_from_user_6x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
	__asm_copy_from_user_4x_cont(to, from, ret,	\
		"	GETW D1Ar1,[%1++]\n"		\
		"4:	SETW [%0++],D1Ar1\n" COPY,	\
		"5:	ADD  %2,%2,#2\n"		\
		"	SETW [%0++],D1Ar1\n" FIXUP,	\
		"	.long 4b,5b\n" TENTRY)

#define __asm_copy_from_user_6(to, from, ret) \
	__asm_copy_from_user_6x_cont(to, from, ret, "", "", "")

#define __asm_copy_from_user_7(to, from, ret) \
	__asm_copy_from_user_6x_cont(to, from, ret,	\
		"	GETB D1Ar1,[%1++]\n"		\
		"6:	SETB [%0++],D1Ar1\n",		\
		"7:	ADD  %2,%2,#1\n"		\
		"	SETB [%0++],D1Ar1\n",		\
		"	.long 6b,7b\n")

#define __asm_copy_from_user_8x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
	__asm_copy_from_user_4x_cont(to, from, ret,	\
		"	GETD D1Ar1,[%1++]\n"		\
		"4:	SETD [%0++],D1Ar1\n" COPY,	\
		"5:	ADD  %2,%2,#4\n"			\
		"	SETD [%0++],D1Ar1\n" FIXUP,		\
		"	.long 4b,5b\n" TENTRY)

#define __asm_copy_from_user_8(to, from, ret) \
	__asm_copy_from_user_8x_cont(to, from, ret, "", "", "")

#define __asm_copy_from_user_9(to, from, ret) \
	__asm_copy_from_user_8x_cont(to, from, ret,	\
		"	GETB D1Ar1,[%1++]\n"		\
		"6:	SETB [%0++],D1Ar1\n",		\
		"7:	ADD  %2,%2,#1\n"		\
		"	SETB [%0++],D1Ar1\n",		\
		"	.long 6b,7b\n")

#define __asm_copy_from_user_10x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
	__asm_copy_from_user_8x_cont(to, from, ret,	\
		"	GETW D1Ar1,[%1++]\n"		\
		"6:	SETW [%0++],D1Ar1\n" COPY,	\
		"7:	ADD  %2,%2,#2\n"		\
		"	SETW [%0++],D1Ar1\n" FIXUP,	\
		"	.long 6b,7b\n" TENTRY)

#define __asm_copy_from_user_10(to, from, ret) \
	__asm_copy_from_user_10x_cont(to, from, ret, "", "", "")

#define __asm_copy_from_user_11(to, from, ret)		\
	__asm_copy_from_user_10x_cont(to, from, ret,	\
		"	GETB D1Ar1,[%1++]\n"		\
		"8:	SETB [%0++],D1Ar1\n",		\
		"9:	ADD  %2,%2,#1\n"		\
		"	SETB [%0++],D1Ar1\n",		\
		"	.long 8b,9b\n")

#define __asm_copy_from_user_12x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
	__asm_copy_from_user_8x_cont(to, from, ret,	\
		"	GETD D1Ar1,[%1++]\n"		\
		"6:	SETD [%0++],D1Ar1\n" COPY,	\
		"7:	ADD  %2,%2,#4\n"		\
		"	SETD [%0++],D1Ar1\n" FIXUP,	\
		"	.long 6b,7b\n" TENTRY)

#define __asm_copy_from_user_12(to, from, ret) \
	__asm_copy_from_user_12x_cont(to, from, ret, "", "", "")

#define __asm_copy_from_user_13(to, from, ret) \
	__asm_copy_from_user_12x_cont(to, from, ret,	\
		"	GETB D1Ar1,[%1++]\n"		\
		"8:	SETB [%0++],D1Ar1\n",		\
		"9:	ADD  %2,%2,#1\n"		\
		"	SETB [%0++],D1Ar1\n",		\
		"	.long 8b,9b\n")

#define __asm_copy_from_user_14x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
	__asm_copy_from_user_12x_cont(to, from, ret,	\
		"	GETW D1Ar1,[%1++]\n"		\
		"8:	SETW [%0++],D1Ar1\n" COPY,	\
		"9:	ADD  %2,%2,#2\n"		\
		"	SETW [%0++],D1Ar1\n" FIXUP,	\
		"	.long 8b,9b\n" TENTRY)

#define __asm_copy_from_user_14(to, from, ret) \
	__asm_copy_from_user_14x_cont(to, from, ret, "", "", "")

#define __asm_copy_from_user_15(to, from, ret) \
	__asm_copy_from_user_14x_cont(to, from, ret,	\
		"	GETB D1Ar1,[%1++]\n"		\
		"10:	SETB [%0++],D1Ar1\n",		\
		"11:	ADD  %2,%2,#1\n"		\
		"	SETB [%0++],D1Ar1\n",		\
		"	.long 10b,11b\n")

#define __asm_copy_from_user_16x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
	__asm_copy_from_user_12x_cont(to, from, ret,	\
		"	GETD D1Ar1,[%1++]\n"		\
		"8:	SETD [%0++],D1Ar1\n" COPY,	\
		"9:	ADD  %2,%2,#4\n"		\
		"	SETD [%0++],D1Ar1\n" FIXUP,	\
		"	.long 8b,9b\n" TENTRY)

#define __asm_copy_from_user_16(to, from, ret) \
	__asm_copy_from_user_16x_cont(to, from, ret, "", "", "")

#define __asm_copy_from_user_8x64(to, from, ret) \
	asm volatile (				\
		"	GETL D0Ar2,D1Ar1,[%1++]\n"	\
		"2:	SETL [%0++],D0Ar2,D1Ar1\n"	\
		"1:\n"					\
		"	.section .fixup,\"ax\"\n"	\
		"	MOV D1Ar1,#0\n"			\
		"	MOV D0Ar2,#0\n"			\
		"3:	ADD  %2,%2,#8\n"		\
		"	SETL [%0++],D0Ar2,D1Ar1\n"	\
		"	MOVT    D0Ar2,#HI(1b)\n"	\
		"	JUMP    D0Ar2,#LO(1b)\n"	\
		"	.previous\n"			\
		"	.section __ex_table,\"a\"\n"	\
		"	.long 2b,3b\n"			\
		"	.previous\n"			\
		: "=a" (to), "=r" (from), "=r" (ret)	\
		: "0" (to), "1" (from), "2" (ret)	\
		: "D1Ar1", "D0Ar2", "memory")

/*	rewind 'from' pointer when a fault occurs
 *
 *	Rationale:
 *		A fault occurs while reading from user buffer, which is the
 *		source. Since the fault is at a single address, we only
 *		need to rewind by 8 bytes.
 *		Since we don't write to kernel buffer until we read first,
 *		the kernel buffer is at the right state and needn't be
 *		corrected.
 */
#define __asm_copy_from_user_64bit_rapf_loop(to, from, ret, n, id)	\
	__asm_copy_user_64bit_rapf_loop(to, from, ret, n, id,		\
		"SUB	%1, %1, #8\n")

/*	rewind 'from' pointer when a fault occurs
 *
 *	Rationale:
 *		A fault occurs while reading from user buffer, which is the
 *		source. Since the fault is at a single address, we only
 *		need to rewind by 4 bytes.
 *		Since we don't write to kernel buffer until we read first,
 *		the kernel buffer is at the right state and needn't be
 *		corrected.
 */
#define __asm_copy_from_user_32bit_rapf_loop(to, from, ret, n, id)	\
	__asm_copy_user_32bit_rapf_loop(to, from, ret, n, id,		\
		"SUB	%1, %1, #4\n")


/* Copy from user to kernel, zeroing the bytes that were inaccessible in
   userland.  The return-value is the number of bytes that were
   inaccessible.  */
unsigned long __copy_user_zeroing(void *pdst, const void __user *psrc,
				  unsigned long n)
{
	register char *dst asm ("A0.2") = pdst;
	register const char __user *src asm ("A1.2") = psrc;
	unsigned long retn = 0;

	if (n == 0)
		return 0;

	if ((unsigned long) src & 1) {
		__asm_copy_from_user_1(dst, src, retn);
		n--;
	}
	if ((unsigned long) dst & 1) {
		/* Worst case - byte copy */
		while (n > 0) {
			__asm_copy_from_user_1(dst, src, retn);
			n--;
			if (retn)
				goto copy_exception_bytes;
		}
	}
	if (((unsigned long) src & 2) && n >= 2) {
		__asm_copy_from_user_2(dst, src, retn);
		n -= 2;
	}
	if ((unsigned long) dst & 2) {
		/* Second worst case - word copy */
		while (n >= 2) {
			__asm_copy_from_user_2(dst, src, retn);
			n -= 2;
			if (retn)
				goto copy_exception_bytes;
		}
	}

	/* We only need one check after the unalignment-adjustments,
	   because if both adjustments were done, either both or
	   neither reference had an exception.  */
	if (retn != 0)
		goto copy_exception_bytes;

#ifdef USE_RAPF
	/* 64 bit copy loop */
	if (!(((unsigned long) src | (unsigned long) dst) & 7)) {
		if (n >= RAPF_MIN_BUF_SIZE) {
			/* Copy using fast 64bit rapf */
			__asm_copy_from_user_64bit_rapf_loop(dst, src, retn,
							n, "64cuz");
		}
		while (n >= 8) {
			__asm_copy_from_user_8x64(dst, src, retn);
			n -= 8;
			if (retn)
				goto copy_exception_bytes;
		}
	}

	if (n >= RAPF_MIN_BUF_SIZE) {
		/* Copy using fast 32bit rapf */
		__asm_copy_from_user_32bit_rapf_loop(dst, src, retn,
						n, "32cuz");
	}
#else
	/* 64 bit copy loop */
	if (!(((unsigned long) src | (unsigned long) dst) & 7)) {
		while (n >= 8) {
			__asm_copy_from_user_8x64(dst, src, retn);
			n -= 8;
			if (retn)
				goto copy_exception_bytes;
		}
	}
#endif

	while (n >= 4) {
		__asm_copy_from_user_4(dst, src, retn);
		n -= 4;

		if (retn)
			goto copy_exception_bytes;
	}

	/* If we get here, there were no memory read faults.  */
	switch (n) {
		/* These copies are at least "naturally aligned" (so we don't
		   have to check each byte), due to the src alignment code.
		   The *_3 case *will* get the correct count for retn.  */
	case 0:
		/* This case deliberately left in (if you have doubts check the
		   generated assembly code).  */
		break;
	case 1:
		__asm_copy_from_user_1(dst, src, retn);
		break;
	case 2:
		__asm_copy_from_user_2(dst, src, retn);
		break;
	case 3:
		__asm_copy_from_user_3(dst, src, retn);
		break;
	}

	/* If we get here, retn correctly reflects the number of failing
	   bytes.  */
	return retn;

 copy_exception_bytes:
	/* We already have "retn" bytes cleared, and need to clear the
	   remaining "n" bytes.  A non-optimized simple byte-for-byte in-line
	   memset is preferred here, since this isn't speed-critical code and
	   we'd rather have this a leaf-function than calling memset.  */
	{
		char *endp;
		for (endp = dst + n; dst < endp; dst++)
			*dst = 0;
	}

	return retn + n;
}
EXPORT_SYMBOL(__copy_user_zeroing);

#define __asm_clear_8x64(to, ret) \
	asm volatile (					\
		"	MOV  D0Ar2,#0\n"		\
		"	MOV  D1Ar1,#0\n"		\
		"	SETL [%0],D0Ar2,D1Ar1\n"	\
		"2:	SETL [%0++],D0Ar2,D1Ar1\n"	\
		"1:\n"					\
		"	.section .fixup,\"ax\"\n"	\
		"3:	ADD  %1,%1,#8\n"		\
		"	MOVT    D0Ar2,#HI(1b)\n"	\
		"	JUMP    D0Ar2,#LO(1b)\n"	\
		"	.previous\n"			\
		"	.section __ex_table,\"a\"\n"	\
		"	.long 2b,3b\n"			\
		"	.previous\n"			\
		: "=r" (to), "=r" (ret)	\
		: "0" (to), "1" (ret)	\
		: "D1Ar1", "D0Ar2", "memory")

/* Zero userspace.  */

#define __asm_clear(to, ret, CLEAR, FIXUP, TENTRY) \
	asm volatile (					\
		"	MOV D1Ar1,#0\n"			\
			CLEAR				\
		"1:\n"					\
		"	.section .fixup,\"ax\"\n"	\
			FIXUP				\
		"	MOVT    D1Ar1,#HI(1b)\n"	\
		"	JUMP    D1Ar1,#LO(1b)\n"	\
		"	.previous\n"			\
		"	.section __ex_table,\"a\"\n"	\
			TENTRY				\
		"	.previous"			\
		: "=r" (to), "=r" (ret)			\
		: "0" (to), "1" (ret)			\
		: "D1Ar1", "memory")

#define __asm_clear_1(to, ret) \
	__asm_clear(to, ret,			\
		"	SETB [%0],D1Ar1\n"	\
		"2:	SETB [%0++],D1Ar1\n",	\
		"3:	ADD  %1,%1,#1\n",	\
		"	.long 2b,3b\n")

#define __asm_clear_2(to, ret) \
	__asm_clear(to, ret,			\
		"	SETW [%0],D1Ar1\n"	\
		"2:	SETW [%0++],D1Ar1\n",	\
		"3:	ADD  %1,%1,#2\n",	\
		"	.long 2b,3b\n")

#define __asm_clear_3(to, ret) \
	__asm_clear(to, ret,			\
		 "2:	SETW [%0++],D1Ar1\n"	\
		 "	SETB [%0],D1Ar1\n"	\
		 "3:	SETB [%0++],D1Ar1\n",	\
		 "4:	ADD  %1,%1,#2\n"	\
		 "5:	ADD  %1,%1,#1\n",	\
		 "	.long 2b,4b\n"		\
		 "	.long 3b,5b\n")

#define __asm_clear_4x_cont(to, ret, CLEAR, FIXUP, TENTRY) \
	__asm_clear(to, ret,				\
		"	SETD [%0],D1Ar1\n"		\
		"2:	SETD [%0++],D1Ar1\n" CLEAR,	\
		"3:	ADD  %1,%1,#4\n" FIXUP,		\
		"	.long 2b,3b\n" TENTRY)

#define __asm_clear_4(to, ret) \
	__asm_clear_4x_cont(to, ret, "", "", "")

#define __asm_clear_8x_cont(to, ret, CLEAR, FIXUP, TENTRY) \
	__asm_clear_4x_cont(to, ret,			\
		"	SETD [%0],D1Ar1\n"		\
		"4:	SETD [%0++],D1Ar1\n" CLEAR,	\
		"5:	ADD  %1,%1,#4\n" FIXUP,		\
		"	.long 4b,5b\n" TENTRY)

#define __asm_clear_8(to, ret) \
	__asm_clear_8x_cont(to, ret, "", "", "")

#define __asm_clear_12x_cont(to, ret, CLEAR, FIXUP, TENTRY) \
	__asm_clear_8x_cont(to, ret,			\
		"	SETD [%0],D1Ar1\n"		\
		"6:	SETD [%0++],D1Ar1\n" CLEAR,	\
		"7:	ADD  %1,%1,#4\n" FIXUP,		\
		"	.long 6b,7b\n" TENTRY)

#define __asm_clear_12(to, ret) \
	__asm_clear_12x_cont(to, ret, "", "", "")

#define __asm_clear_16x_cont(to, ret, CLEAR, FIXUP, TENTRY) \
	__asm_clear_12x_cont(to, ret,			\
		"	SETD [%0],D1Ar1\n"		\
		"8:	SETD [%0++],D1Ar1\n" CLEAR,	\
		"9:	ADD  %1,%1,#4\n" FIXUP,		\
		"	.long 8b,9b\n" TENTRY)

#define __asm_clear_16(to, ret) \
	__asm_clear_16x_cont(to, ret, "", "", "")

unsigned long __do_clear_user(void __user *pto, unsigned long pn)
{
	register char __user *dst asm ("D0Re0") = pto;
	register unsigned long n asm ("D1Re0") = pn;
	register unsigned long retn asm ("D0Ar6") = 0;

	if ((unsigned long) dst & 1) {
		__asm_clear_1(dst, retn);
		n--;
	}

	if ((unsigned long) dst & 2) {
		__asm_clear_2(dst, retn);
		n -= 2;
	}

	/* 64 bit copy loop */
	if (!((__force unsigned long) dst & 7)) {
		while (n >= 8) {
			__asm_clear_8x64(dst, retn);
			n -= 8;
		}
	}

	while (n >= 16) {
		__asm_clear_16(dst, retn);
		n -= 16;
	}

	while (n >= 4) {
		__asm_clear_4(dst, retn);
		n -= 4;
	}

	switch (n) {
	case 0:
		break;
	case 1:
		__asm_clear_1(dst, retn);
		break;
	case 2:
		__asm_clear_2(dst, retn);
		break;
	case 3:
		__asm_clear_3(dst, retn);
		break;
	}

	return retn;
}
EXPORT_SYMBOL(__do_clear_user);

unsigned char __get_user_asm_b(const void __user *addr, long *err)
{
	register unsigned char x asm ("D0Re0") = 0;
	asm volatile (
		"	GETB %0,[%2]\n"
		"1:\n"
		"	GETB %0,[%2]\n"
		"2:\n"
		"	.section .fixup,\"ax\"\n"
		"3:	MOV     D0FrT,%3\n"
		"	SETD    [%1],D0FrT\n"
		"	MOVT    D0FrT,#HI(2b)\n"
		"	JUMP    D0FrT,#LO(2b)\n"
		"	.previous\n"
		"	.section __ex_table,\"a\"\n"
		"	.long 1b,3b\n"
		"	.previous\n"
		: "=r" (x)
		: "r" (err), "r" (addr), "P" (-EFAULT)
		: "D0FrT");
	return x;
}
EXPORT_SYMBOL(__get_user_asm_b);

unsigned short __get_user_asm_w(const void __user *addr, long *err)
{
	register unsigned short x asm ("D0Re0") = 0;
	asm volatile (
		"	GETW %0,[%2]\n"
		"1:\n"
		"	GETW %0,[%2]\n"
		"2:\n"
		"	.section .fixup,\"ax\"\n"
		"3:	MOV     D0FrT,%3\n"
		"	SETD    [%1],D0FrT\n"
		"	MOVT    D0FrT,#HI(2b)\n"
		"	JUMP    D0FrT,#LO(2b)\n"
		"	.previous\n"
		"	.section __ex_table,\"a\"\n"
		"	.long 1b,3b\n"
		"	.previous\n"
		: "=r" (x)
		: "r" (err), "r" (addr), "P" (-EFAULT)
		: "D0FrT");
	return x;
}
EXPORT_SYMBOL(__get_user_asm_w);

unsigned int __get_user_asm_d(const void __user *addr, long *err)
{
	register unsigned int x asm ("D0Re0") = 0;
	asm volatile (
		"	GETD %0,[%2]\n"
		"1:\n"
		"	GETD %0,[%2]\n"
		"2:\n"
		"	.section .fixup,\"ax\"\n"
		"3:	MOV     D0FrT,%3\n"
		"	SETD    [%1],D0FrT\n"
		"	MOVT    D0FrT,#HI(2b)\n"
		"	JUMP    D0FrT,#LO(2b)\n"
		"	.previous\n"
		"	.section __ex_table,\"a\"\n"
		"	.long 1b,3b\n"
		"	.previous\n"
		: "=r" (x)
		: "r" (err), "r" (addr), "P" (-EFAULT)
		: "D0FrT");
	return x;
}
EXPORT_SYMBOL(__get_user_asm_d);

long __put_user_asm_b(unsigned int x, void __user *addr)
{
	register unsigned int err asm ("D0Re0") = 0;
	asm volatile (
		"	MOV  %0,#0\n"
		"	SETB [%2],%1\n"
		"1:\n"
		"	SETB [%2],%1\n"
		"2:\n"
		".section .fixup,\"ax\"\n"
		"3:	MOV     %0,%3\n"
		"	MOVT    D0FrT,#HI(2b)\n"
		"	JUMP    D0FrT,#LO(2b)\n"
		".previous\n"
		".section __ex_table,\"a\"\n"
		"	.long 1b,3b\n"
		".previous"
		: "=r"(err)
		: "d" (x), "a" (addr), "P"(-EFAULT)
		: "D0FrT");
	return err;
}
EXPORT_SYMBOL(__put_user_asm_b);

long __put_user_asm_w(unsigned int x, void __user *addr)
{
	register unsigned int err asm ("D0Re0") = 0;
	asm volatile (
		"	MOV  %0,#0\n"
		"	SETW [%2],%1\n"
		"1:\n"
		"	SETW [%2],%1\n"
		"2:\n"
		".section .fixup,\"ax\"\n"
		"3:	MOV     %0,%3\n"
		"	MOVT    D0FrT,#HI(2b)\n"
		"	JUMP    D0FrT,#LO(2b)\n"
		".previous\n"
		".section __ex_table,\"a\"\n"
		"	.long 1b,3b\n"
		".previous"
		: "=r"(err)
		: "d" (x), "a" (addr), "P"(-EFAULT)
		: "D0FrT");
	return err;
}
EXPORT_SYMBOL(__put_user_asm_w);

long __put_user_asm_d(unsigned int x, void __user *addr)
{
	register unsigned int err asm ("D0Re0") = 0;
	asm volatile (
		"	MOV  %0,#0\n"
		"	SETD [%2],%1\n"
		"1:\n"
		"	SETD [%2],%1\n"
		"2:\n"
		".section .fixup,\"ax\"\n"
		"3:	MOV     %0,%3\n"
		"	MOVT    D0FrT,#HI(2b)\n"
		"	JUMP    D0FrT,#LO(2b)\n"
		".previous\n"
		".section __ex_table,\"a\"\n"
		"	.long 1b,3b\n"
		".previous"
		: "=r"(err)
		: "d" (x), "a" (addr), "P"(-EFAULT)
		: "D0FrT");
	return err;
}
EXPORT_SYMBOL(__put_user_asm_d);

long __put_user_asm_l(unsigned long long x, void __user *addr)
{
	register unsigned int err asm ("D0Re0") = 0;
	asm volatile (
		"	MOV  %0,#0\n"
		"	SETL [%2],%1,%t1\n"
		"1:\n"
		"	SETL [%2],%1,%t1\n"
		"2:\n"
		".section .fixup,\"ax\"\n"
		"3:	MOV     %0,%3\n"
		"	MOVT    D0FrT,#HI(2b)\n"
		"	JUMP    D0FrT,#LO(2b)\n"
		".previous\n"
		".section __ex_table,\"a\"\n"
		"	.long 1b,3b\n"
		".previous"
		: "=r"(err)
		: "d" (x), "a" (addr), "P"(-EFAULT)
		: "D0FrT");
	return err;
}
EXPORT_SYMBOL(__put_user_asm_l);

long strnlen_user(const char __user *src, long count)
{
	long res;

	if (!access_ok(VERIFY_READ, src, 0))
		return 0;

	asm volatile ("	MOV     D0Ar4, %1\n"
		      "	MOV     D0Ar6, %2\n"
		      "0:\n"
		      "	SUBS    D0FrT, D0Ar6, #0\n"
		      "	SUB     D0Ar6, D0Ar6, #1\n"
		      "	BLE     2f\n"
		      "	GETB    D0FrT, [D0Ar4+#1++]\n"
		      "1:\n"
		      "	TST     D0FrT, #255\n"
		      "	BNE     0b\n"
		      "2:\n"
		      "	SUB     %0, %2, D0Ar6\n"
		      "3:\n"
		      "	.section .fixup,\"ax\"\n"
		      "4:\n"
		      "	MOV     %0, #0\n"
		      "	MOVT    D0FrT,#HI(3b)\n"
		      "	JUMP    D0FrT,#LO(3b)\n"
		      "	.previous\n"
		      "	.section __ex_table,\"a\"\n"
		      "	.long 1b,4b\n"
		      "	.previous\n"
		      : "=r" (res)
		      : "r" (src), "r" (count)
		      : "D0FrT", "D0Ar4", "D0Ar6", "cc");

	return res;
}
EXPORT_SYMBOL(strnlen_user);

long __strncpy_from_user(char *dst, const char __user *src, long count)
{
	long res;

	if (count == 0)
		return 0;

	/*
	 * Currently, in 2.4.0-test9, most ports use a simple byte-copy loop.
	 *  So do we.
	 *
	 *  This code is deduced from:
	 *
	 *      char tmp2;
	 *      long tmp1, tmp3;
	 *      tmp1 = count;
	 *      while ((*dst++ = (tmp2 = *src++)) != 0
	 *             && --tmp1)
	 *        ;
	 *
	 *      res = count - tmp1;
	 *
	 *  with tweaks.
	 */

	asm volatile ("	MOV  %0,%3\n"
		      "1:\n"
		      "	GETB D0FrT,[%2++]\n"
		      "2:\n"
		      "	CMP  D0FrT,#0\n"
		      "	SETB [%1++],D0FrT\n"
		      "	BEQ  3f\n"
		      "	SUBS %0,%0,#1\n"
		      "	BNZ  1b\n"
		      "3:\n"
		      "	SUB  %0,%3,%0\n"
		      "4:\n"
		      "	.section .fixup,\"ax\"\n"
		      "5:\n"
		      "	MOV  %0,%7\n"
		      "	MOVT    D0FrT,#HI(4b)\n"
		      "	JUMP    D0FrT,#LO(4b)\n"
		      "	.previous\n"
		      "	.section __ex_table,\"a\"\n"
		      "	.long 2b,5b\n"
		      "	.previous"
		      : "=r" (res), "=r" (dst), "=r" (src), "=r" (count)
		      : "3" (count), "1" (dst), "2" (src), "P" (-EFAULT)
		      : "D0FrT", "memory", "cc");

	return res;
}
EXPORT_SYMBOL(__strncpy_from_user);