1/* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * Copyright (C) 1996, 1997, 1998, 1999, 2000, 03, 04 by Ralf Baechle 7 * Copyright (C) 1999, 2000 Silicon Graphics, Inc. 8 * Copyright (C) 2007 Maciej W. Rozycki 9 */ 10#ifndef _ASM_UACCESS_H 11#define _ASM_UACCESS_H 12 13#include <linux/kernel.h> 14#include <linux/errno.h> 15#include <linux/thread_info.h> 16 17/* 18 * The fs value determines whether argument validity checking should be 19 * performed or not. If get_fs() == USER_DS, checking is performed, with 20 * get_fs() == KERNEL_DS, checking is bypassed. 21 * 22 * For historical reasons, these macros are grossly misnamed. 23 */ 24#ifdef CONFIG_32BIT 25 26#define __UA_LIMIT 0x80000000UL 27 28#define __UA_ADDR ".word" 29#define __UA_LA "la" 30#define __UA_ADDU "addu" 31#define __UA_t0 "$8" 32#define __UA_t1 "$9" 33 34#endif /* CONFIG_32BIT */ 35 36#ifdef CONFIG_64BIT 37 38extern u64 __ua_limit; 39 40#define __UA_LIMIT __ua_limit 41 42#define __UA_ADDR ".dword" 43#define __UA_LA "dla" 44#define __UA_ADDU "daddu" 45#define __UA_t0 "$12" 46#define __UA_t1 "$13" 47 48#endif /* CONFIG_64BIT */ 49 50/* 51 * USER_DS is a bitmask that has the bits set that may not be set in a valid 52 * userspace address. Note that we limit 32-bit userspace to 0x7fff8000 but 53 * the arithmetic we're doing only works if the limit is a power of two, so 54 * we use 0x80000000 here on 32-bit kernels. If a process passes an invalid 55 * address in this range it's the process's problem, not ours :-) 56 */ 57 58#define KERNEL_DS ((mm_segment_t) { 0UL }) 59#define USER_DS ((mm_segment_t) { __UA_LIMIT }) 60 61#define VERIFY_READ 0 62#define VERIFY_WRITE 1 63 64#define get_ds() (KERNEL_DS) 65#define get_fs() (current_thread_info()->addr_limit) 66#define set_fs(x) (current_thread_info()->addr_limit = (x)) 67 68#define segment_eq(a, b) ((a).seg == (b).seg) 69 70 71/* 72 * Is a address valid? This does a straighforward calculation rather 73 * than tests. 74 * 75 * Address valid if: 76 * - "addr" doesn't have any high-bits set 77 * - AND "size" doesn't have any high-bits set 78 * - AND "addr+size" doesn't have any high-bits set 79 * - OR we are in kernel mode. 80 * 81 * __ua_size() is a trick to avoid runtime checking of positive constant 82 * sizes; for those we already know at compile time that the size is ok. 83 */ 84#define __ua_size(size) \ 85 ((__builtin_constant_p(size) && (signed long) (size) > 0) ? 0 : (size)) 86 87/* 88 * access_ok: - Checks if a user space pointer is valid 89 * @type: Type of access: %VERIFY_READ or %VERIFY_WRITE. Note that 90 * %VERIFY_WRITE is a superset of %VERIFY_READ - if it is safe 91 * to write to a block, it is always safe to read from it. 92 * @addr: User space pointer to start of block to check 93 * @size: Size of block to check 94 * 95 * Context: User context only. This function may sleep. 96 * 97 * Checks if a pointer to a block of memory in user space is valid. 98 * 99 * Returns true (nonzero) if the memory block may be valid, false (zero) 100 * if it is definitely invalid. 101 * 102 * Note that, depending on architecture, this function probably just 103 * checks that the pointer is in the user space range - after calling 104 * this function, memory access functions may still return -EFAULT. 105 */ 106 107#define __access_mask get_fs().seg 108 109#define __access_ok(addr, size, mask) \ 110({ \ 111 unsigned long __addr = (unsigned long) (addr); \ 112 unsigned long __size = size; \ 113 unsigned long __mask = mask; \ 114 unsigned long __ok; \ 115 \ 116 __chk_user_ptr(addr); \ 117 __ok = (signed long)(__mask & (__addr | (__addr + __size) | \ 118 __ua_size(__size))); \ 119 __ok == 0; \ 120}) 121 122#define access_ok(type, addr, size) \ 123 likely(__access_ok((addr), (size), __access_mask)) 124 125/* 126 * put_user: - Write a simple value into user space. 127 * @x: Value to copy to user space. 128 * @ptr: Destination address, in user space. 129 * 130 * Context: User context only. This function may sleep. 131 * 132 * This macro copies a single simple value from kernel space to user 133 * space. It supports simple types like char and int, but not larger 134 * data types like structures or arrays. 135 * 136 * @ptr must have pointer-to-simple-variable type, and @x must be assignable 137 * to the result of dereferencing @ptr. 138 * 139 * Returns zero on success, or -EFAULT on error. 140 */ 141#define put_user(x,ptr) \ 142 __put_user_check((x), (ptr), sizeof(*(ptr))) 143 144/* 145 * get_user: - Get a simple variable from user space. 146 * @x: Variable to store result. 147 * @ptr: Source address, in user space. 148 * 149 * Context: User context only. This function may sleep. 150 * 151 * This macro copies a single simple variable from user space to kernel 152 * space. It supports simple types like char and int, but not larger 153 * data types like structures or arrays. 154 * 155 * @ptr must have pointer-to-simple-variable type, and the result of 156 * dereferencing @ptr must be assignable to @x without a cast. 157 * 158 * Returns zero on success, or -EFAULT on error. 159 * On error, the variable @x is set to zero. 160 */ 161#define get_user(x,ptr) \ 162 __get_user_check((x), (ptr), sizeof(*(ptr))) 163 164/* 165 * __put_user: - Write a simple value into user space, with less checking. 166 * @x: Value to copy to user space. 167 * @ptr: Destination address, in user space. 168 * 169 * Context: User context only. This function may sleep. 170 * 171 * This macro copies a single simple value from kernel space to user 172 * space. It supports simple types like char and int, but not larger 173 * data types like structures or arrays. 174 * 175 * @ptr must have pointer-to-simple-variable type, and @x must be assignable 176 * to the result of dereferencing @ptr. 177 * 178 * Caller must check the pointer with access_ok() before calling this 179 * function. 180 * 181 * Returns zero on success, or -EFAULT on error. 182 */ 183#define __put_user(x,ptr) \ 184 __put_user_nocheck((x), (ptr), sizeof(*(ptr))) 185 186/* 187 * __get_user: - Get a simple variable from user space, with less checking. 188 * @x: Variable to store result. 189 * @ptr: Source address, in user space. 190 * 191 * Context: User context only. This function may sleep. 192 * 193 * This macro copies a single simple variable from user space to kernel 194 * space. It supports simple types like char and int, but not larger 195 * data types like structures or arrays. 196 * 197 * @ptr must have pointer-to-simple-variable type, and the result of 198 * dereferencing @ptr must be assignable to @x without a cast. 199 * 200 * Caller must check the pointer with access_ok() before calling this 201 * function. 202 * 203 * Returns zero on success, or -EFAULT on error. 204 * On error, the variable @x is set to zero. 205 */ 206#define __get_user(x,ptr) \ 207 __get_user_nocheck((x), (ptr), sizeof(*(ptr))) 208 209struct __large_struct { unsigned long buf[100]; }; 210#define __m(x) (*(struct __large_struct __user *)(x)) 211 212/* 213 * Yuck. We need two variants, one for 64bit operation and one 214 * for 32 bit mode and old iron. 215 */ 216#ifdef CONFIG_32BIT 217#define __GET_USER_DW(val, ptr) __get_user_asm_ll32(val, ptr) 218#endif 219#ifdef CONFIG_64BIT 220#define __GET_USER_DW(val, ptr) __get_user_asm(val, "ld", ptr) 221#endif 222 223extern void __get_user_unknown(void); 224 225#define __get_user_common(val, size, ptr) \ 226do { \ 227 switch (size) { \ 228 case 1: __get_user_asm(val, "lb", ptr); break; \ 229 case 2: __get_user_asm(val, "lh", ptr); break; \ 230 case 4: __get_user_asm(val, "lw", ptr); break; \ 231 case 8: __GET_USER_DW(val, ptr); break; \ 232 default: __get_user_unknown(); break; \ 233 } \ 234} while (0) 235 236#define __get_user_nocheck(x, ptr, size) \ 237({ \ 238 int __gu_err; \ 239 \ 240 __chk_user_ptr(ptr); \ 241 __get_user_common((x), size, ptr); \ 242 __gu_err; \ 243}) 244 245#define __get_user_check(x, ptr, size) \ 246({ \ 247 int __gu_err = -EFAULT; \ 248 const __typeof__(*(ptr)) __user * __gu_ptr = (ptr); \ 249 \ 250 might_fault(); \ 251 if (likely(access_ok(VERIFY_READ, __gu_ptr, size))) \ 252 __get_user_common((x), size, __gu_ptr); \ 253 \ 254 __gu_err; \ 255}) 256 257#define __get_user_asm(val, insn, addr) \ 258{ \ 259 long __gu_tmp; \ 260 \ 261 __asm__ __volatile__( \ 262 "1: " insn " %1, %3 \n" \ 263 "2: \n" \ 264 " .section .fixup,\"ax\" \n" \ 265 "3: li %0, %4 \n" \ 266 " j 2b \n" \ 267 " .previous \n" \ 268 " .section __ex_table,\"a\" \n" \ 269 " "__UA_ADDR "\t1b, 3b \n" \ 270 " .previous \n" \ 271 : "=r" (__gu_err), "=r" (__gu_tmp) \ 272 : "0" (0), "o" (__m(addr)), "i" (-EFAULT)); \ 273 \ 274 (val) = (__typeof__(*(addr))) __gu_tmp; \ 275} 276 277/* 278 * Get a long long 64 using 32 bit registers. 279 */ 280#define __get_user_asm_ll32(val, addr) \ 281{ \ 282 union { \ 283 unsigned long long l; \ 284 __typeof__(*(addr)) t; \ 285 } __gu_tmp; \ 286 \ 287 __asm__ __volatile__( \ 288 "1: lw %1, (%3) \n" \ 289 "2: lw %D1, 4(%3) \n" \ 290 "3: .section .fixup,\"ax\" \n" \ 291 "4: li %0, %4 \n" \ 292 " move %1, $0 \n" \ 293 " move %D1, $0 \n" \ 294 " j 3b \n" \ 295 " .previous \n" \ 296 " .section __ex_table,\"a\" \n" \ 297 " " __UA_ADDR " 1b, 4b \n" \ 298 " " __UA_ADDR " 2b, 4b \n" \ 299 " .previous \n" \ 300 : "=r" (__gu_err), "=&r" (__gu_tmp.l) \ 301 : "0" (0), "r" (addr), "i" (-EFAULT)); \ 302 \ 303 (val) = __gu_tmp.t; \ 304} 305 306/* 307 * Yuck. We need two variants, one for 64bit operation and one 308 * for 32 bit mode and old iron. 309 */ 310#ifdef CONFIG_32BIT 311#define __PUT_USER_DW(ptr) __put_user_asm_ll32(ptr) 312#endif 313#ifdef CONFIG_64BIT 314#define __PUT_USER_DW(ptr) __put_user_asm("sd", ptr) 315#endif 316 317#define __put_user_nocheck(x, ptr, size) \ 318({ \ 319 __typeof__(*(ptr)) __pu_val; \ 320 int __pu_err = 0; \ 321 \ 322 __chk_user_ptr(ptr); \ 323 __pu_val = (x); \ 324 switch (size) { \ 325 case 1: __put_user_asm("sb", ptr); break; \ 326 case 2: __put_user_asm("sh", ptr); break; \ 327 case 4: __put_user_asm("sw", ptr); break; \ 328 case 8: __PUT_USER_DW(ptr); break; \ 329 default: __put_user_unknown(); break; \ 330 } \ 331 __pu_err; \ 332}) 333 334#define __put_user_check(x, ptr, size) \ 335({ \ 336 __typeof__(*(ptr)) __user *__pu_addr = (ptr); \ 337 __typeof__(*(ptr)) __pu_val = (x); \ 338 int __pu_err = -EFAULT; \ 339 \ 340 might_fault(); \ 341 if (likely(access_ok(VERIFY_WRITE, __pu_addr, size))) { \ 342 switch (size) { \ 343 case 1: __put_user_asm("sb", __pu_addr); break; \ 344 case 2: __put_user_asm("sh", __pu_addr); break; \ 345 case 4: __put_user_asm("sw", __pu_addr); break; \ 346 case 8: __PUT_USER_DW(__pu_addr); break; \ 347 default: __put_user_unknown(); break; \ 348 } \ 349 } \ 350 __pu_err; \ 351}) 352 353#define __put_user_asm(insn, ptr) \ 354{ \ 355 __asm__ __volatile__( \ 356 "1: " insn " %z2, %3 # __put_user_asm\n" \ 357 "2: \n" \ 358 " .section .fixup,\"ax\" \n" \ 359 "3: li %0, %4 \n" \ 360 " j 2b \n" \ 361 " .previous \n" \ 362 " .section __ex_table,\"a\" \n" \ 363 " " __UA_ADDR " 1b, 3b \n" \ 364 " .previous \n" \ 365 : "=r" (__pu_err) \ 366 : "0" (0), "Jr" (__pu_val), "o" (__m(ptr)), \ 367 "i" (-EFAULT)); \ 368} 369 370#define __put_user_asm_ll32(ptr) \ 371{ \ 372 __asm__ __volatile__( \ 373 "1: sw %2, (%3) # __put_user_asm_ll32 \n" \ 374 "2: sw %D2, 4(%3) \n" \ 375 "3: \n" \ 376 " .section .fixup,\"ax\" \n" \ 377 "4: li %0, %4 \n" \ 378 " j 3b \n" \ 379 " .previous \n" \ 380 " .section __ex_table,\"a\" \n" \ 381 " " __UA_ADDR " 1b, 4b \n" \ 382 " " __UA_ADDR " 2b, 4b \n" \ 383 " .previous" \ 384 : "=r" (__pu_err) \ 385 : "0" (0), "r" (__pu_val), "r" (ptr), \ 386 "i" (-EFAULT)); \ 387} 388 389extern void __put_user_unknown(void); 390 391/* 392 * put_user_unaligned: - Write a simple value into user space. 393 * @x: Value to copy to user space. 394 * @ptr: Destination address, in user space. 395 * 396 * Context: User context only. This function may sleep. 397 * 398 * This macro copies a single simple value from kernel space to user 399 * space. It supports simple types like char and int, but not larger 400 * data types like structures or arrays. 401 * 402 * @ptr must have pointer-to-simple-variable type, and @x must be assignable 403 * to the result of dereferencing @ptr. 404 * 405 * Returns zero on success, or -EFAULT on error. 406 */ 407#define put_user_unaligned(x,ptr) \ 408 __put_user_unaligned_check((x),(ptr),sizeof(*(ptr))) 409 410/* 411 * get_user_unaligned: - Get a simple variable from user space. 412 * @x: Variable to store result. 413 * @ptr: Source address, in user space. 414 * 415 * Context: User context only. This function may sleep. 416 * 417 * This macro copies a single simple variable from user space to kernel 418 * space. It supports simple types like char and int, but not larger 419 * data types like structures or arrays. 420 * 421 * @ptr must have pointer-to-simple-variable type, and the result of 422 * dereferencing @ptr must be assignable to @x without a cast. 423 * 424 * Returns zero on success, or -EFAULT on error. 425 * On error, the variable @x is set to zero. 426 */ 427#define get_user_unaligned(x,ptr) \ 428 __get_user_unaligned_check((x),(ptr),sizeof(*(ptr))) 429 430/* 431 * __put_user_unaligned: - Write a simple value into user space, with less checking. 432 * @x: Value to copy to user space. 433 * @ptr: Destination address, in user space. 434 * 435 * Context: User context only. This function may sleep. 436 * 437 * This macro copies a single simple value from kernel space to user 438 * space. It supports simple types like char and int, but not larger 439 * data types like structures or arrays. 440 * 441 * @ptr must have pointer-to-simple-variable type, and @x must be assignable 442 * to the result of dereferencing @ptr. 443 * 444 * Caller must check the pointer with access_ok() before calling this 445 * function. 446 * 447 * Returns zero on success, or -EFAULT on error. 448 */ 449#define __put_user_unaligned(x,ptr) \ 450 __put_user_unaligned_nocheck((x),(ptr),sizeof(*(ptr))) 451 452/* 453 * __get_user_unaligned: - Get a simple variable from user space, with less checking. 454 * @x: Variable to store result. 455 * @ptr: Source address, in user space. 456 * 457 * Context: User context only. This function may sleep. 458 * 459 * This macro copies a single simple variable from user space to kernel 460 * space. It supports simple types like char and int, but not larger 461 * data types like structures or arrays. 462 * 463 * @ptr must have pointer-to-simple-variable type, and the result of 464 * dereferencing @ptr must be assignable to @x without a cast. 465 * 466 * Caller must check the pointer with access_ok() before calling this 467 * function. 468 * 469 * Returns zero on success, or -EFAULT on error. 470 * On error, the variable @x is set to zero. 471 */ 472#define __get_user_unaligned(x,ptr) \ 473 __get_user__unalignednocheck((x),(ptr),sizeof(*(ptr))) 474 475/* 476 * Yuck. We need two variants, one for 64bit operation and one 477 * for 32 bit mode and old iron. 478 */ 479#ifdef CONFIG_32BIT 480#define __GET_USER_UNALIGNED_DW(val, ptr) \ 481 __get_user_unaligned_asm_ll32(val, ptr) 482#endif 483#ifdef CONFIG_64BIT 484#define __GET_USER_UNALIGNED_DW(val, ptr) \ 485 __get_user_unaligned_asm(val, "uld", ptr) 486#endif 487 488extern void __get_user_unaligned_unknown(void); 489 490#define __get_user_unaligned_common(val, size, ptr) \ 491do { \ 492 switch (size) { \ 493 case 1: __get_user_asm(val, "lb", ptr); break; \ 494 case 2: __get_user_unaligned_asm(val, "ulh", ptr); break; \ 495 case 4: __get_user_unaligned_asm(val, "ulw", ptr); break; \ 496 case 8: __GET_USER_UNALIGNED_DW(val, ptr); break; \ 497 default: __get_user_unaligned_unknown(); break; \ 498 } \ 499} while (0) 500 501#define __get_user_unaligned_nocheck(x,ptr,size) \ 502({ \ 503 int __gu_err; \ 504 \ 505 __get_user_unaligned_common((x), size, ptr); \ 506 __gu_err; \ 507}) 508 509#define __get_user_unaligned_check(x,ptr,size) \ 510({ \ 511 int __gu_err = -EFAULT; \ 512 const __typeof__(*(ptr)) __user * __gu_ptr = (ptr); \ 513 \ 514 if (likely(access_ok(VERIFY_READ, __gu_ptr, size))) \ 515 __get_user_unaligned_common((x), size, __gu_ptr); \ 516 \ 517 __gu_err; \ 518}) 519 520#define __get_user_unaligned_asm(val, insn, addr) \ 521{ \ 522 long __gu_tmp; \ 523 \ 524 __asm__ __volatile__( \ 525 "1: " insn " %1, %3 \n" \ 526 "2: \n" \ 527 " .section .fixup,\"ax\" \n" \ 528 "3: li %0, %4 \n" \ 529 " j 2b \n" \ 530 " .previous \n" \ 531 " .section __ex_table,\"a\" \n" \ 532 " "__UA_ADDR "\t1b, 3b \n" \ 533 " "__UA_ADDR "\t1b + 4, 3b \n" \ 534 " .previous \n" \ 535 : "=r" (__gu_err), "=r" (__gu_tmp) \ 536 : "0" (0), "o" (__m(addr)), "i" (-EFAULT)); \ 537 \ 538 (val) = (__typeof__(*(addr))) __gu_tmp; \ 539} 540 541/* 542 * Get a long long 64 using 32 bit registers. 543 */ 544#define __get_user_unaligned_asm_ll32(val, addr) \ 545{ \ 546 unsigned long long __gu_tmp; \ 547 \ 548 __asm__ __volatile__( \ 549 "1: ulw %1, (%3) \n" \ 550 "2: ulw %D1, 4(%3) \n" \ 551 " move %0, $0 \n" \ 552 "3: .section .fixup,\"ax\" \n" \ 553 "4: li %0, %4 \n" \ 554 " move %1, $0 \n" \ 555 " move %D1, $0 \n" \ 556 " j 3b \n" \ 557 " .previous \n" \ 558 " .section __ex_table,\"a\" \n" \ 559 " " __UA_ADDR " 1b, 4b \n" \ 560 " " __UA_ADDR " 1b + 4, 4b \n" \ 561 " " __UA_ADDR " 2b, 4b \n" \ 562 " " __UA_ADDR " 2b + 4, 4b \n" \ 563 " .previous \n" \ 564 : "=r" (__gu_err), "=&r" (__gu_tmp) \ 565 : "0" (0), "r" (addr), "i" (-EFAULT)); \ 566 (val) = (__typeof__(*(addr))) __gu_tmp; \ 567} 568 569/* 570 * Yuck. We need two variants, one for 64bit operation and one 571 * for 32 bit mode and old iron. 572 */ 573#ifdef CONFIG_32BIT 574#define __PUT_USER_UNALIGNED_DW(ptr) __put_user_unaligned_asm_ll32(ptr) 575#endif 576#ifdef CONFIG_64BIT 577#define __PUT_USER_UNALIGNED_DW(ptr) __put_user_unaligned_asm("usd", ptr) 578#endif 579 580#define __put_user_unaligned_nocheck(x,ptr,size) \ 581({ \ 582 __typeof__(*(ptr)) __pu_val; \ 583 int __pu_err = 0; \ 584 \ 585 __pu_val = (x); \ 586 switch (size) { \ 587 case 1: __put_user_asm("sb", ptr); break; \ 588 case 2: __put_user_unaligned_asm("ush", ptr); break; \ 589 case 4: __put_user_unaligned_asm("usw", ptr); break; \ 590 case 8: __PUT_USER_UNALIGNED_DW(ptr); break; \ 591 default: __put_user_unaligned_unknown(); break; \ 592 } \ 593 __pu_err; \ 594}) 595 596#define __put_user_unaligned_check(x,ptr,size) \ 597({ \ 598 __typeof__(*(ptr)) __user *__pu_addr = (ptr); \ 599 __typeof__(*(ptr)) __pu_val = (x); \ 600 int __pu_err = -EFAULT; \ 601 \ 602 if (likely(access_ok(VERIFY_WRITE, __pu_addr, size))) { \ 603 switch (size) { \ 604 case 1: __put_user_asm("sb", __pu_addr); break; \ 605 case 2: __put_user_unaligned_asm("ush", __pu_addr); break; \ 606 case 4: __put_user_unaligned_asm("usw", __pu_addr); break; \ 607 case 8: __PUT_USER_UNALGINED_DW(__pu_addr); break; \ 608 default: __put_user_unaligned_unknown(); break; \ 609 } \ 610 } \ 611 __pu_err; \ 612}) 613 614#define __put_user_unaligned_asm(insn, ptr) \ 615{ \ 616 __asm__ __volatile__( \ 617 "1: " insn " %z2, %3 # __put_user_unaligned_asm\n" \ 618 "2: \n" \ 619 " .section .fixup,\"ax\" \n" \ 620 "3: li %0, %4 \n" \ 621 " j 2b \n" \ 622 " .previous \n" \ 623 " .section __ex_table,\"a\" \n" \ 624 " " __UA_ADDR " 1b, 3b \n" \ 625 " .previous \n" \ 626 : "=r" (__pu_err) \ 627 : "0" (0), "Jr" (__pu_val), "o" (__m(ptr)), \ 628 "i" (-EFAULT)); \ 629} 630 631#define __put_user_unaligned_asm_ll32(ptr) \ 632{ \ 633 __asm__ __volatile__( \ 634 "1: sw %2, (%3) # __put_user_unaligned_asm_ll32 \n" \ 635 "2: sw %D2, 4(%3) \n" \ 636 "3: \n" \ 637 " .section .fixup,\"ax\" \n" \ 638 "4: li %0, %4 \n" \ 639 " j 3b \n" \ 640 " .previous \n" \ 641 " .section __ex_table,\"a\" \n" \ 642 " " __UA_ADDR " 1b, 4b \n" \ 643 " " __UA_ADDR " 1b + 4, 4b \n" \ 644 " " __UA_ADDR " 2b, 4b \n" \ 645 " " __UA_ADDR " 2b + 4, 4b \n" \ 646 " .previous" \ 647 : "=r" (__pu_err) \ 648 : "0" (0), "r" (__pu_val), "r" (ptr), \ 649 "i" (-EFAULT)); \ 650} 651 652extern void __put_user_unaligned_unknown(void); 653 654/* 655 * We're generating jump to subroutines which will be outside the range of 656 * jump instructions 657 */ 658#ifdef MODULE 659#define __MODULE_JAL(destination) \ 660 ".set\tnoat\n\t" \ 661 __UA_LA "\t$1, " #destination "\n\t" \ 662 "jalr\t$1\n\t" \ 663 ".set\tat\n\t" 664#else 665#define __MODULE_JAL(destination) \ 666 "jal\t" #destination "\n\t" 667#endif 668 669#ifndef CONFIG_CPU_DADDI_WORKAROUNDS 670#define DADDI_SCRATCH "$0" 671#else 672#define DADDI_SCRATCH "$3" 673#endif 674 675extern size_t __copy_user(void *__to, const void *__from, size_t __n); 676 677#define __invoke_copy_to_user(to, from, n) \ 678({ \ 679 register void __user *__cu_to_r __asm__("$4"); \ 680 register const void *__cu_from_r __asm__("$5"); \ 681 register long __cu_len_r __asm__("$6"); \ 682 \ 683 __cu_to_r = (to); \ 684 __cu_from_r = (from); \ 685 __cu_len_r = (n); \ 686 __asm__ __volatile__( \ 687 __MODULE_JAL(__copy_user) \ 688 : "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r) \ 689 : \ 690 : "$8", "$9", "$10", "$11", "$12", "$15", "$24", "$31", \ 691 DADDI_SCRATCH, "memory"); \ 692 __cu_len_r; \ 693}) 694 695/* 696 * __copy_to_user: - Copy a block of data into user space, with less checking. 697 * @to: Destination address, in user space. 698 * @from: Source address, in kernel space. 699 * @n: Number of bytes to copy. 700 * 701 * Context: User context only. This function may sleep. 702 * 703 * Copy data from kernel space to user space. Caller must check 704 * the specified block with access_ok() before calling this function. 705 * 706 * Returns number of bytes that could not be copied. 707 * On success, this will be zero. 708 */ 709#define __copy_to_user(to, from, n) \ 710({ \ 711 void __user *__cu_to; \ 712 const void *__cu_from; \ 713 long __cu_len; \ 714 \ 715 __cu_to = (to); \ 716 __cu_from = (from); \ 717 __cu_len = (n); \ 718 might_fault(); \ 719 __cu_len = __invoke_copy_to_user(__cu_to, __cu_from, __cu_len); \ 720 __cu_len; \ 721}) 722 723extern size_t __copy_user_inatomic(void *__to, const void *__from, size_t __n); 724 725#define __copy_to_user_inatomic(to, from, n) \ 726({ \ 727 void __user *__cu_to; \ 728 const void *__cu_from; \ 729 long __cu_len; \ 730 \ 731 __cu_to = (to); \ 732 __cu_from = (from); \ 733 __cu_len = (n); \ 734 __cu_len = __invoke_copy_to_user(__cu_to, __cu_from, __cu_len); \ 735 __cu_len; \ 736}) 737 738#define __copy_from_user_inatomic(to, from, n) \ 739({ \ 740 void *__cu_to; \ 741 const void __user *__cu_from; \ 742 long __cu_len; \ 743 \ 744 __cu_to = (to); \ 745 __cu_from = (from); \ 746 __cu_len = (n); \ 747 __cu_len = __invoke_copy_from_user_inatomic(__cu_to, __cu_from, \ 748 __cu_len); \ 749 __cu_len; \ 750}) 751 752/* 753 * copy_to_user: - Copy a block of data into user space. 754 * @to: Destination address, in user space. 755 * @from: Source address, in kernel space. 756 * @n: Number of bytes to copy. 757 * 758 * Context: User context only. This function may sleep. 759 * 760 * Copy data from kernel space to user space. 761 * 762 * Returns number of bytes that could not be copied. 763 * On success, this will be zero. 764 */ 765#define copy_to_user(to, from, n) \ 766({ \ 767 void __user *__cu_to; \ 768 const void *__cu_from; \ 769 long __cu_len; \ 770 \ 771 __cu_to = (to); \ 772 __cu_from = (from); \ 773 __cu_len = (n); \ 774 if (access_ok(VERIFY_WRITE, __cu_to, __cu_len)) { \ 775 might_fault(); \ 776 __cu_len = __invoke_copy_to_user(__cu_to, __cu_from, \ 777 __cu_len); \ 778 } \ 779 __cu_len; \ 780}) 781 782#define __invoke_copy_from_user(to, from, n) \ 783({ \ 784 register void *__cu_to_r __asm__("$4"); \ 785 register const void __user *__cu_from_r __asm__("$5"); \ 786 register long __cu_len_r __asm__("$6"); \ 787 \ 788 __cu_to_r = (to); \ 789 __cu_from_r = (from); \ 790 __cu_len_r = (n); \ 791 __asm__ __volatile__( \ 792 ".set\tnoreorder\n\t" \ 793 __MODULE_JAL(__copy_user) \ 794 ".set\tnoat\n\t" \ 795 __UA_ADDU "\t$1, %1, %2\n\t" \ 796 ".set\tat\n\t" \ 797 ".set\treorder" \ 798 : "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r) \ 799 : \ 800 : "$8", "$9", "$10", "$11", "$12", "$15", "$24", "$31", \ 801 DADDI_SCRATCH, "memory"); \ 802 __cu_len_r; \ 803}) 804 805#define __invoke_copy_from_user_inatomic(to, from, n) \ 806({ \ 807 register void *__cu_to_r __asm__("$4"); \ 808 register const void __user *__cu_from_r __asm__("$5"); \ 809 register long __cu_len_r __asm__("$6"); \ 810 \ 811 __cu_to_r = (to); \ 812 __cu_from_r = (from); \ 813 __cu_len_r = (n); \ 814 __asm__ __volatile__( \ 815 ".set\tnoreorder\n\t" \ 816 __MODULE_JAL(__copy_user_inatomic) \ 817 ".set\tnoat\n\t" \ 818 __UA_ADDU "\t$1, %1, %2\n\t" \ 819 ".set\tat\n\t" \ 820 ".set\treorder" \ 821 : "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r) \ 822 : \ 823 : "$8", "$9", "$10", "$11", "$12", "$15", "$24", "$31", \ 824 DADDI_SCRATCH, "memory"); \ 825 __cu_len_r; \ 826}) 827 828/* 829 * __copy_from_user: - Copy a block of data from user space, with less checking. 830 * @to: Destination address, in kernel space. 831 * @from: Source address, in user space. 832 * @n: Number of bytes to copy. 833 * 834 * Context: User context only. This function may sleep. 835 * 836 * Copy data from user space to kernel space. Caller must check 837 * the specified block with access_ok() before calling this function. 838 * 839 * Returns number of bytes that could not be copied. 840 * On success, this will be zero. 841 * 842 * If some data could not be copied, this function will pad the copied 843 * data to the requested size using zero bytes. 844 */ 845#define __copy_from_user(to, from, n) \ 846({ \ 847 void *__cu_to; \ 848 const void __user *__cu_from; \ 849 long __cu_len; \ 850 \ 851 __cu_to = (to); \ 852 __cu_from = (from); \ 853 __cu_len = (n); \ 854 might_fault(); \ 855 __cu_len = __invoke_copy_from_user(__cu_to, __cu_from, \ 856 __cu_len); \ 857 __cu_len; \ 858}) 859 860/* 861 * copy_from_user: - Copy a block of data from user space. 862 * @to: Destination address, in kernel space. 863 * @from: Source address, in user space. 864 * @n: Number of bytes to copy. 865 * 866 * Context: User context only. This function may sleep. 867 * 868 * Copy data from user space to kernel space. 869 * 870 * Returns number of bytes that could not be copied. 871 * On success, this will be zero. 872 * 873 * If some data could not be copied, this function will pad the copied 874 * data to the requested size using zero bytes. 875 */ 876#define copy_from_user(to, from, n) \ 877({ \ 878 void *__cu_to; \ 879 const void __user *__cu_from; \ 880 long __cu_len; \ 881 \ 882 __cu_to = (to); \ 883 __cu_from = (from); \ 884 __cu_len = (n); \ 885 if (access_ok(VERIFY_READ, __cu_from, __cu_len)) { \ 886 might_fault(); \ 887 __cu_len = __invoke_copy_from_user(__cu_to, __cu_from, \ 888 __cu_len); \ 889 } \ 890 __cu_len; \ 891}) 892 893#define __copy_in_user(to, from, n) \ 894({ \ 895 void __user *__cu_to; \ 896 const void __user *__cu_from; \ 897 long __cu_len; \ 898 \ 899 __cu_to = (to); \ 900 __cu_from = (from); \ 901 __cu_len = (n); \ 902 might_fault(); \ 903 __cu_len = __invoke_copy_from_user(__cu_to, __cu_from, \ 904 __cu_len); \ 905 __cu_len; \ 906}) 907 908#define copy_in_user(to, from, n) \ 909({ \ 910 void __user *__cu_to; \ 911 const void __user *__cu_from; \ 912 long __cu_len; \ 913 \ 914 __cu_to = (to); \ 915 __cu_from = (from); \ 916 __cu_len = (n); \ 917 if (likely(access_ok(VERIFY_READ, __cu_from, __cu_len) && \ 918 access_ok(VERIFY_WRITE, __cu_to, __cu_len))) { \ 919 might_fault(); \ 920 __cu_len = __invoke_copy_from_user(__cu_to, __cu_from, \ 921 __cu_len); \ 922 } \ 923 __cu_len; \ 924}) 925 926/* 927 * __clear_user: - Zero a block of memory in user space, with less checking. 928 * @to: Destination address, in user space. 929 * @n: Number of bytes to zero. 930 * 931 * Zero a block of memory in user space. Caller must check 932 * the specified block with access_ok() before calling this function. 933 * 934 * Returns number of bytes that could not be cleared. 935 * On success, this will be zero. 936 */ 937static inline __kernel_size_t 938__clear_user(void __user *addr, __kernel_size_t size) 939{ 940 __kernel_size_t res; 941 942 might_fault(); 943 __asm__ __volatile__( 944 "move\t$4, %1\n\t" 945 "move\t$5, $0\n\t" 946 "move\t$6, %2\n\t" 947 __MODULE_JAL(__bzero) 948 "move\t%0, $6" 949 : "=r" (res) 950 : "r" (addr), "r" (size) 951 : "$4", "$5", "$6", __UA_t0, __UA_t1, "$31"); 952 953 return res; 954} 955 956#define clear_user(addr,n) \ 957({ \ 958 void __user * __cl_addr = (addr); \ 959 unsigned long __cl_size = (n); \ 960 if (__cl_size && access_ok(VERIFY_WRITE, \ 961 __cl_addr, __cl_size)) \ 962 __cl_size = __clear_user(__cl_addr, __cl_size); \ 963 __cl_size; \ 964}) 965 966/* 967 * __strncpy_from_user: - Copy a NUL terminated string from userspace, with less checking. 968 * @dst: Destination address, in kernel space. This buffer must be at 969 * least @count bytes long. 970 * @src: Source address, in user space. 971 * @count: Maximum number of bytes to copy, including the trailing NUL. 972 * 973 * Copies a NUL-terminated string from userspace to kernel space. 974 * Caller must check the specified block with access_ok() before calling 975 * this function. 976 * 977 * On success, returns the length of the string (not including the trailing 978 * NUL). 979 * 980 * If access to userspace fails, returns -EFAULT (some data may have been 981 * copied). 982 * 983 * If @count is smaller than the length of the string, copies @count bytes 984 * and returns @count. 985 */ 986static inline long 987__strncpy_from_user(char *__to, const char __user *__from, long __len) 988{ 989 long res; 990 991 might_fault(); 992 __asm__ __volatile__( 993 "move\t$4, %1\n\t" 994 "move\t$5, %2\n\t" 995 "move\t$6, %3\n\t" 996 __MODULE_JAL(__strncpy_from_user_nocheck_asm) 997 "move\t%0, $2" 998 : "=r" (res) 999 : "r" (__to), "r" (__from), "r" (__len) 1000 : "$2", "$3", "$4", "$5", "$6", __UA_t0, "$31", "memory"); 1001 1002 return res; 1003} 1004 1005/* 1006 * strncpy_from_user: - Copy a NUL terminated string from userspace. 1007 * @dst: Destination address, in kernel space. This buffer must be at 1008 * least @count bytes long. 1009 * @src: Source address, in user space. 1010 * @count: Maximum number of bytes to copy, including the trailing NUL. 1011 * 1012 * Copies a NUL-terminated string from userspace to kernel space. 1013 * 1014 * On success, returns the length of the string (not including the trailing 1015 * NUL). 1016 * 1017 * If access to userspace fails, returns -EFAULT (some data may have been 1018 * copied). 1019 * 1020 * If @count is smaller than the length of the string, copies @count bytes 1021 * and returns @count. 1022 */ 1023static inline long 1024strncpy_from_user(char *__to, const char __user *__from, long __len) 1025{ 1026 long res; 1027 1028 might_fault(); 1029 __asm__ __volatile__( 1030 "move\t$4, %1\n\t" 1031 "move\t$5, %2\n\t" 1032 "move\t$6, %3\n\t" 1033 __MODULE_JAL(__strncpy_from_user_asm) 1034 "move\t%0, $2" 1035 : "=r" (res) 1036 : "r" (__to), "r" (__from), "r" (__len) 1037 : "$2", "$3", "$4", "$5", "$6", __UA_t0, "$31", "memory"); 1038 1039 return res; 1040} 1041 1042/* Returns: 0 if bad, string length+1 (memory size) of string if ok */ 1043static inline long __strlen_user(const char __user *s) 1044{ 1045 long res; 1046 1047 might_fault(); 1048 __asm__ __volatile__( 1049 "move\t$4, %1\n\t" 1050 __MODULE_JAL(__strlen_user_nocheck_asm) 1051 "move\t%0, $2" 1052 : "=r" (res) 1053 : "r" (s) 1054 : "$2", "$4", __UA_t0, "$31"); 1055 1056 return res; 1057} 1058 1059/* 1060 * strlen_user: - Get the size of a string in user space. 1061 * @str: The string to measure. 1062 * 1063 * Context: User context only. This function may sleep. 1064 * 1065 * Get the size of a NUL-terminated string in user space. 1066 * 1067 * Returns the size of the string INCLUDING the terminating NUL. 1068 * On exception, returns 0. 1069 * 1070 * If there is a limit on the length of a valid string, you may wish to 1071 * consider using strnlen_user() instead. 1072 */ 1073static inline long strlen_user(const char __user *s) 1074{ 1075 long res; 1076 1077 might_fault(); 1078 __asm__ __volatile__( 1079 "move\t$4, %1\n\t" 1080 __MODULE_JAL(__strlen_user_asm) 1081 "move\t%0, $2" 1082 : "=r" (res) 1083 : "r" (s) 1084 : "$2", "$4", __UA_t0, "$31"); 1085 1086 return res; 1087} 1088 1089/* Returns: 0 if bad, string length+1 (memory size) of string if ok */ 1090static inline long __strnlen_user(const char __user *s, long n) 1091{ 1092 long res; 1093 1094 might_fault(); 1095 __asm__ __volatile__( 1096 "move\t$4, %1\n\t" 1097 "move\t$5, %2\n\t" 1098 __MODULE_JAL(__strnlen_user_nocheck_asm) 1099 "move\t%0, $2" 1100 : "=r" (res) 1101 : "r" (s), "r" (n) 1102 : "$2", "$4", "$5", __UA_t0, "$31"); 1103 1104 return res; 1105} 1106 1107/* 1108 * strlen_user: - Get the size of a string in user space. 1109 * @str: The string to measure. 1110 * 1111 * Context: User context only. This function may sleep. 1112 * 1113 * Get the size of a NUL-terminated string in user space. 1114 * 1115 * Returns the size of the string INCLUDING the terminating NUL. 1116 * On exception, returns 0. 1117 * 1118 * If there is a limit on the length of a valid string, you may wish to 1119 * consider using strnlen_user() instead. 1120 */ 1121static inline long strnlen_user(const char __user *s, long n) 1122{ 1123 long res; 1124 1125 might_fault(); 1126 __asm__ __volatile__( 1127 "move\t$4, %1\n\t" 1128 "move\t$5, %2\n\t" 1129 __MODULE_JAL(__strnlen_user_asm) 1130 "move\t%0, $2" 1131 : "=r" (res) 1132 : "r" (s), "r" (n) 1133 : "$2", "$4", "$5", __UA_t0, "$31"); 1134 1135 return res; 1136} 1137 1138struct exception_table_entry 1139{ 1140 unsigned long insn; 1141 unsigned long nextinsn; 1142}; 1143 1144extern int fixup_exception(struct pt_regs *regs); 1145 1146#endif /* _ASM_UACCESS_H */ 1147