1/* 2 * comedi_buf.c 3 * 4 * COMEDI - Linux Control and Measurement Device Interface 5 * Copyright (C) 1997-2000 David A. Schleef <ds@schleef.org> 6 * Copyright (C) 2002 Frank Mori Hess <fmhess@users.sourceforge.net> 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License as published by 10 * the Free Software Foundation; either version 2 of the License, or 11 * (at your option) any later version. 12 * 13 * This program is distributed in the hope that it will be useful, 14 * but WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 * GNU General Public License for more details. 17 */ 18 19#include <linux/vmalloc.h> 20#include <linux/slab.h> 21 22#include "comedidev.h" 23#include "comedi_internal.h" 24 25#ifdef PAGE_KERNEL_NOCACHE 26#define COMEDI_PAGE_PROTECTION PAGE_KERNEL_NOCACHE 27#else 28#define COMEDI_PAGE_PROTECTION PAGE_KERNEL 29#endif 30 31static void comedi_buf_map_kref_release(struct kref *kref) 32{ 33 struct comedi_buf_map *bm = 34 container_of(kref, struct comedi_buf_map, refcount); 35 struct comedi_buf_page *buf; 36 unsigned int i; 37 38 if (bm->page_list) { 39 for (i = 0; i < bm->n_pages; i++) { 40 buf = &bm->page_list[i]; 41 clear_bit(PG_reserved, 42 &(virt_to_page(buf->virt_addr)->flags)); 43 if (bm->dma_dir != DMA_NONE) { 44#ifdef CONFIG_HAS_DMA 45 dma_free_coherent(bm->dma_hw_dev, 46 PAGE_SIZE, 47 buf->virt_addr, 48 buf->dma_addr); 49#endif 50 } else { 51 free_page((unsigned long)buf->virt_addr); 52 } 53 } 54 vfree(bm->page_list); 55 } 56 if (bm->dma_dir != DMA_NONE) 57 put_device(bm->dma_hw_dev); 58 kfree(bm); 59} 60 61static void __comedi_buf_free(struct comedi_device *dev, 62 struct comedi_subdevice *s) 63{ 64 struct comedi_async *async = s->async; 65 struct comedi_buf_map *bm; 66 unsigned long flags; 67 68 if (async->prealloc_buf) { 69 vunmap(async->prealloc_buf); 70 async->prealloc_buf = NULL; 71 async->prealloc_bufsz = 0; 72 } 73 74 spin_lock_irqsave(&s->spin_lock, flags); 75 bm = async->buf_map; 76 async->buf_map = NULL; 77 spin_unlock_irqrestore(&s->spin_lock, flags); 78 comedi_buf_map_put(bm); 79} 80 81static void __comedi_buf_alloc(struct comedi_device *dev, 82 struct comedi_subdevice *s, 83 unsigned n_pages) 84{ 85 struct comedi_async *async = s->async; 86 struct page **pages = NULL; 87 struct comedi_buf_map *bm; 88 struct comedi_buf_page *buf; 89 unsigned long flags; 90 unsigned i; 91 92 if (!IS_ENABLED(CONFIG_HAS_DMA) && s->async_dma_dir != DMA_NONE) { 93 dev_err(dev->class_dev, 94 "dma buffer allocation not supported\n"); 95 return; 96 } 97 98 bm = kzalloc(sizeof(*async->buf_map), GFP_KERNEL); 99 if (!bm) 100 return; 101 102 kref_init(&bm->refcount); 103 spin_lock_irqsave(&s->spin_lock, flags); 104 async->buf_map = bm; 105 spin_unlock_irqrestore(&s->spin_lock, flags); 106 bm->dma_dir = s->async_dma_dir; 107 if (bm->dma_dir != DMA_NONE) 108 /* Need ref to hardware device to free buffer later. */ 109 bm->dma_hw_dev = get_device(dev->hw_dev); 110 111 bm->page_list = vzalloc(sizeof(*buf) * n_pages); 112 if (bm->page_list) 113 pages = vmalloc(sizeof(struct page *) * n_pages); 114 115 if (!pages) 116 return; 117 118 for (i = 0; i < n_pages; i++) { 119 buf = &bm->page_list[i]; 120 if (bm->dma_dir != DMA_NONE) 121#ifdef CONFIG_HAS_DMA 122 buf->virt_addr = dma_alloc_coherent(bm->dma_hw_dev, 123 PAGE_SIZE, 124 &buf->dma_addr, 125 GFP_KERNEL | 126 __GFP_COMP); 127#else 128 break; 129#endif 130 else 131 buf->virt_addr = (void *)get_zeroed_page(GFP_KERNEL); 132 if (!buf->virt_addr) 133 break; 134 135 set_bit(PG_reserved, &(virt_to_page(buf->virt_addr)->flags)); 136 137 pages[i] = virt_to_page(buf->virt_addr); 138 } 139 spin_lock_irqsave(&s->spin_lock, flags); 140 bm->n_pages = i; 141 spin_unlock_irqrestore(&s->spin_lock, flags); 142 143 /* vmap the prealloc_buf if all the pages were allocated */ 144 if (i == n_pages) 145 async->prealloc_buf = vmap(pages, n_pages, VM_MAP, 146 COMEDI_PAGE_PROTECTION); 147 148 vfree(pages); 149} 150 151void comedi_buf_map_get(struct comedi_buf_map *bm) 152{ 153 if (bm) 154 kref_get(&bm->refcount); 155} 156 157int comedi_buf_map_put(struct comedi_buf_map *bm) 158{ 159 if (bm) 160 return kref_put(&bm->refcount, comedi_buf_map_kref_release); 161 return 1; 162} 163 164/* returns s->async->buf_map and increments its kref refcount */ 165struct comedi_buf_map * 166comedi_buf_map_from_subdev_get(struct comedi_subdevice *s) 167{ 168 struct comedi_async *async = s->async; 169 struct comedi_buf_map *bm = NULL; 170 unsigned long flags; 171 172 if (!async) 173 return NULL; 174 175 spin_lock_irqsave(&s->spin_lock, flags); 176 bm = async->buf_map; 177 /* only want it if buffer pages allocated */ 178 if (bm && bm->n_pages) 179 comedi_buf_map_get(bm); 180 else 181 bm = NULL; 182 spin_unlock_irqrestore(&s->spin_lock, flags); 183 184 return bm; 185} 186 187bool comedi_buf_is_mmapped(struct comedi_subdevice *s) 188{ 189 struct comedi_buf_map *bm = s->async->buf_map; 190 191 return bm && (atomic_read(&bm->refcount.refcount) > 1); 192} 193 194int comedi_buf_alloc(struct comedi_device *dev, struct comedi_subdevice *s, 195 unsigned long new_size) 196{ 197 struct comedi_async *async = s->async; 198 199 /* Round up new_size to multiple of PAGE_SIZE */ 200 new_size = (new_size + PAGE_SIZE - 1) & PAGE_MASK; 201 202 /* if no change is required, do nothing */ 203 if (async->prealloc_buf && async->prealloc_bufsz == new_size) 204 return 0; 205 206 /* deallocate old buffer */ 207 __comedi_buf_free(dev, s); 208 209 /* allocate new buffer */ 210 if (new_size) { 211 unsigned n_pages = new_size >> PAGE_SHIFT; 212 213 __comedi_buf_alloc(dev, s, n_pages); 214 215 if (!async->prealloc_buf) { 216 /* allocation failed */ 217 __comedi_buf_free(dev, s); 218 return -ENOMEM; 219 } 220 } 221 async->prealloc_bufsz = new_size; 222 223 return 0; 224} 225 226void comedi_buf_reset(struct comedi_subdevice *s) 227{ 228 struct comedi_async *async = s->async; 229 230 async->buf_write_alloc_count = 0; 231 async->buf_write_count = 0; 232 async->buf_read_alloc_count = 0; 233 async->buf_read_count = 0; 234 235 async->buf_write_ptr = 0; 236 async->buf_read_ptr = 0; 237 238 async->cur_chan = 0; 239 async->scan_progress = 0; 240 async->munge_chan = 0; 241 async->munge_count = 0; 242 async->munge_ptr = 0; 243 244 async->events = 0; 245} 246 247static unsigned int comedi_buf_write_n_available(struct comedi_subdevice *s) 248{ 249 struct comedi_async *async = s->async; 250 unsigned int free_end = async->buf_read_count + async->prealloc_bufsz; 251 252 return free_end - async->buf_write_alloc_count; 253} 254 255static unsigned int __comedi_buf_write_alloc(struct comedi_subdevice *s, 256 unsigned int nbytes, 257 int strict) 258{ 259 struct comedi_async *async = s->async; 260 unsigned int available = comedi_buf_write_n_available(s); 261 262 if (nbytes > available) 263 nbytes = strict ? 0 : available; 264 265 async->buf_write_alloc_count += nbytes; 266 267 /* 268 * ensure the async buffer 'counts' are read and updated 269 * before we write data to the write-alloc'ed buffer space 270 */ 271 smp_mb(); 272 273 return nbytes; 274} 275 276/* allocates chunk for the writer from free buffer space */ 277unsigned int comedi_buf_write_alloc(struct comedi_subdevice *s, 278 unsigned int nbytes) 279{ 280 return __comedi_buf_write_alloc(s, nbytes, 0); 281} 282EXPORT_SYMBOL_GPL(comedi_buf_write_alloc); 283 284/* 285 * munging is applied to data by core as it passes between user 286 * and kernel space 287 */ 288static unsigned int comedi_buf_munge(struct comedi_subdevice *s, 289 unsigned int num_bytes) 290{ 291 struct comedi_async *async = s->async; 292 unsigned int count = 0; 293 const unsigned num_sample_bytes = bytes_per_sample(s); 294 295 if (!s->munge || (async->cmd.flags & CMDF_RAWDATA)) { 296 async->munge_count += num_bytes; 297 count = num_bytes; 298 } else { 299 /* don't munge partial samples */ 300 num_bytes -= num_bytes % num_sample_bytes; 301 while (count < num_bytes) { 302 int block_size = num_bytes - count; 303 unsigned int buf_end; 304 305 buf_end = async->prealloc_bufsz - async->munge_ptr; 306 if (block_size > buf_end) 307 block_size = buf_end; 308 309 s->munge(s->device, s, 310 async->prealloc_buf + async->munge_ptr, 311 block_size, async->munge_chan); 312 313 /* 314 * ensure data is munged in buffer before the 315 * async buffer munge_count is incremented 316 */ 317 smp_wmb(); 318 319 async->munge_chan += block_size / num_sample_bytes; 320 async->munge_chan %= async->cmd.chanlist_len; 321 async->munge_count += block_size; 322 async->munge_ptr += block_size; 323 async->munge_ptr %= async->prealloc_bufsz; 324 count += block_size; 325 } 326 } 327 328 return count; 329} 330 331unsigned int comedi_buf_write_n_allocated(struct comedi_subdevice *s) 332{ 333 struct comedi_async *async = s->async; 334 335 return async->buf_write_alloc_count - async->buf_write_count; 336} 337 338/* transfers a chunk from writer to filled buffer space */ 339unsigned int comedi_buf_write_free(struct comedi_subdevice *s, 340 unsigned int nbytes) 341{ 342 struct comedi_async *async = s->async; 343 unsigned int allocated = comedi_buf_write_n_allocated(s); 344 345 if (nbytes > allocated) 346 nbytes = allocated; 347 348 async->buf_write_count += nbytes; 349 async->buf_write_ptr += nbytes; 350 comedi_buf_munge(s, async->buf_write_count - async->munge_count); 351 if (async->buf_write_ptr >= async->prealloc_bufsz) 352 async->buf_write_ptr %= async->prealloc_bufsz; 353 354 return nbytes; 355} 356EXPORT_SYMBOL_GPL(comedi_buf_write_free); 357 358unsigned int comedi_buf_read_n_available(struct comedi_subdevice *s) 359{ 360 struct comedi_async *async = s->async; 361 unsigned num_bytes; 362 363 if (!async) 364 return 0; 365 366 num_bytes = async->munge_count - async->buf_read_count; 367 368 /* 369 * ensure the async buffer 'counts' are read before we 370 * attempt to read data from the buffer 371 */ 372 smp_rmb(); 373 374 return num_bytes; 375} 376EXPORT_SYMBOL_GPL(comedi_buf_read_n_available); 377 378/* allocates a chunk for the reader from filled (and munged) buffer space */ 379unsigned int comedi_buf_read_alloc(struct comedi_subdevice *s, 380 unsigned int nbytes) 381{ 382 struct comedi_async *async = s->async; 383 unsigned int available; 384 385 available = async->munge_count - async->buf_read_alloc_count; 386 if (nbytes > available) 387 nbytes = available; 388 389 async->buf_read_alloc_count += nbytes; 390 391 /* 392 * ensure the async buffer 'counts' are read before we 393 * attempt to read data from the read-alloc'ed buffer space 394 */ 395 smp_rmb(); 396 397 return nbytes; 398} 399EXPORT_SYMBOL_GPL(comedi_buf_read_alloc); 400 401static unsigned int comedi_buf_read_n_allocated(struct comedi_async *async) 402{ 403 return async->buf_read_alloc_count - async->buf_read_count; 404} 405 406/* transfers control of a chunk from reader to free buffer space */ 407unsigned int comedi_buf_read_free(struct comedi_subdevice *s, 408 unsigned int nbytes) 409{ 410 struct comedi_async *async = s->async; 411 unsigned int allocated; 412 413 /* 414 * ensure data has been read out of buffer before 415 * the async read count is incremented 416 */ 417 smp_mb(); 418 419 allocated = comedi_buf_read_n_allocated(async); 420 if (nbytes > allocated) 421 nbytes = allocated; 422 423 async->buf_read_count += nbytes; 424 async->buf_read_ptr += nbytes; 425 async->buf_read_ptr %= async->prealloc_bufsz; 426 return nbytes; 427} 428EXPORT_SYMBOL_GPL(comedi_buf_read_free); 429 430int comedi_buf_put(struct comedi_subdevice *s, unsigned short x) 431{ 432 struct comedi_async *async = s->async; 433 unsigned int n = __comedi_buf_write_alloc(s, sizeof(short), 1); 434 435 if (n < sizeof(short)) { 436 async->events |= COMEDI_CB_ERROR; 437 return 0; 438 } 439 *(unsigned short *)(async->prealloc_buf + async->buf_write_ptr) = x; 440 comedi_buf_write_free(s, sizeof(short)); 441 return 1; 442} 443EXPORT_SYMBOL_GPL(comedi_buf_put); 444 445int comedi_buf_get(struct comedi_subdevice *s, unsigned short *x) 446{ 447 struct comedi_async *async = s->async; 448 unsigned int n = comedi_buf_read_n_available(s); 449 450 if (n < sizeof(short)) 451 return 0; 452 comedi_buf_read_alloc(s, sizeof(short)); 453 *x = *(unsigned short *)(async->prealloc_buf + async->buf_read_ptr); 454 comedi_buf_read_free(s, sizeof(short)); 455 return 1; 456} 457EXPORT_SYMBOL_GPL(comedi_buf_get); 458 459void comedi_buf_memcpy_to(struct comedi_subdevice *s, unsigned int offset, 460 const void *data, unsigned int num_bytes) 461{ 462 struct comedi_async *async = s->async; 463 unsigned int write_ptr = async->buf_write_ptr + offset; 464 465 if (write_ptr >= async->prealloc_bufsz) 466 write_ptr %= async->prealloc_bufsz; 467 468 while (num_bytes) { 469 unsigned int block_size; 470 471 if (write_ptr + num_bytes > async->prealloc_bufsz) 472 block_size = async->prealloc_bufsz - write_ptr; 473 else 474 block_size = num_bytes; 475 476 memcpy(async->prealloc_buf + write_ptr, data, block_size); 477 478 data += block_size; 479 num_bytes -= block_size; 480 481 write_ptr = 0; 482 } 483} 484EXPORT_SYMBOL_GPL(comedi_buf_memcpy_to); 485 486void comedi_buf_memcpy_from(struct comedi_subdevice *s, unsigned int offset, 487 void *dest, unsigned int nbytes) 488{ 489 void *src; 490 struct comedi_async *async = s->async; 491 unsigned int read_ptr = async->buf_read_ptr + offset; 492 493 if (read_ptr >= async->prealloc_bufsz) 494 read_ptr %= async->prealloc_bufsz; 495 496 while (nbytes) { 497 unsigned int block_size; 498 499 src = async->prealloc_buf + read_ptr; 500 501 if (nbytes >= async->prealloc_bufsz - read_ptr) 502 block_size = async->prealloc_bufsz - read_ptr; 503 else 504 block_size = nbytes; 505 506 memcpy(dest, src, block_size); 507 nbytes -= block_size; 508 dest += block_size; 509 read_ptr = 0; 510 } 511} 512EXPORT_SYMBOL_GPL(comedi_buf_memcpy_from); 513 514/** 515 * comedi_write_array_to_buffer - write data to comedi buffer 516 * @s: comedi_subdevice struct 517 * @data: destination 518 * @num_bytes: number of bytes to write 519 * 520 * Writes up to num_bytes bytes of data to the comedi buffer associated with 521 * the subdevice, marks it as written and updates the acquisition scan 522 * progress. 523 * 524 * Returns the amount of data written in bytes. 525 */ 526unsigned int comedi_write_array_to_buffer(struct comedi_subdevice *s, 527 const void *data, 528 unsigned int num_bytes) 529{ 530 struct comedi_async *async = s->async; 531 unsigned int retval; 532 533 if (num_bytes == 0) 534 return 0; 535 536 retval = comedi_buf_write_alloc(s, num_bytes); 537 if (retval != num_bytes) { 538 dev_warn(s->device->class_dev, "buffer overrun\n"); 539 async->events |= COMEDI_CB_OVERFLOW; 540 return 0; 541 } 542 543 comedi_buf_memcpy_to(s, 0, data, num_bytes); 544 comedi_buf_write_free(s, num_bytes); 545 comedi_inc_scan_progress(s, num_bytes); 546 async->events |= COMEDI_CB_BLOCK; 547 548 return num_bytes; 549} 550EXPORT_SYMBOL_GPL(comedi_write_array_to_buffer); 551 552/** 553 * comedi_read_array_from_buffer - read data from comedi buffer 554 * @s: comedi_subdevice struct 555 * @data: destination 556 * @num_bytes: number of bytes to read 557 * 558 * Reads up to num_bytes bytes of data from the comedi buffer associated with 559 * the subdevice, marks it as read and updates the acquisition scan progress. 560 * 561 * Returns the amount of data read in bytes. 562 */ 563unsigned int comedi_read_array_from_buffer(struct comedi_subdevice *s, 564 void *data, unsigned int num_bytes) 565{ 566 if (num_bytes == 0) 567 return 0; 568 569 num_bytes = comedi_buf_read_alloc(s, num_bytes); 570 comedi_buf_memcpy_from(s, 0, data, num_bytes); 571 comedi_buf_read_free(s, num_bytes); 572 comedi_inc_scan_progress(s, num_bytes); 573 s->async->events |= COMEDI_CB_BLOCK; 574 575 return num_bytes; 576} 577EXPORT_SYMBOL_GPL(comedi_read_array_from_buffer); 578