1/* 2 * CARMA DATA-FPGA Access Driver 3 * 4 * Copyright (c) 2009-2011 Ira W. Snyder <iws@ovro.caltech.edu> 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License as published by the 8 * Free Software Foundation; either version 2 of the License, or (at your 9 * option) any later version. 10 */ 11 12/* 13 * FPGA Memory Dump Format 14 * 15 * FPGA #0 control registers (32 x 32-bit words) 16 * FPGA #1 control registers (32 x 32-bit words) 17 * FPGA #2 control registers (32 x 32-bit words) 18 * FPGA #3 control registers (32 x 32-bit words) 19 * SYSFPGA control registers (32 x 32-bit words) 20 * FPGA #0 correlation array (NUM_CORL0 correlation blocks) 21 * FPGA #1 correlation array (NUM_CORL1 correlation blocks) 22 * FPGA #2 correlation array (NUM_CORL2 correlation blocks) 23 * FPGA #3 correlation array (NUM_CORL3 correlation blocks) 24 * 25 * Each correlation array consists of: 26 * 27 * Correlation Data (2 x NUM_LAGSn x 32-bit words) 28 * Pipeline Metadata (2 x NUM_METAn x 32-bit words) 29 * Quantization Counters (2 x NUM_QCNTn x 32-bit words) 30 * 31 * The NUM_CORLn, NUM_LAGSn, NUM_METAn, and NUM_QCNTn values come from 32 * the FPGA configuration registers. They do not change once the FPGA's 33 * have been programmed, they only change on re-programming. 34 */ 35 36/* 37 * Basic Description: 38 * 39 * This driver is used to capture correlation spectra off of the four data 40 * processing FPGAs. The FPGAs are often reprogrammed at runtime, therefore 41 * this driver supports dynamic enable/disable of capture while the device 42 * remains open. 43 * 44 * The nominal capture rate is 64Hz (every 15.625ms). To facilitate this fast 45 * capture rate, all buffers are pre-allocated to avoid any potentially long 46 * running memory allocations while capturing. 47 * 48 * There are two lists and one pointer which are used to keep track of the 49 * different states of data buffers. 50 * 51 * 1) free list 52 * This list holds all empty data buffers which are ready to receive data. 53 * 54 * 2) inflight pointer 55 * This pointer holds the currently inflight data buffer. This buffer is having 56 * data copied into it by the DMA engine. 57 * 58 * 3) used list 59 * This list holds data buffers which have been filled, and are waiting to be 60 * read by userspace. 61 * 62 * All buffers start life on the free list, then move successively to the 63 * inflight pointer, and then to the used list. After they have been read by 64 * userspace, they are moved back to the free list. The cycle repeats as long 65 * as necessary. 66 * 67 * It should be noted that all buffers are mapped and ready for DMA when they 68 * are on any of the three lists. They are only unmapped when they are in the 69 * process of being read by userspace. 70 */ 71 72/* 73 * Notes on the IRQ masking scheme: 74 * 75 * The IRQ masking scheme here is different than most other hardware. The only 76 * way for the DATA-FPGAs to detect if the kernel has taken too long to copy 77 * the data is if the status registers are not cleared before the next 78 * correlation data dump is ready. 79 * 80 * The interrupt line is connected to the status registers, such that when they 81 * are cleared, the interrupt is de-asserted. Therein lies our problem. We need 82 * to schedule a long-running DMA operation and return from the interrupt 83 * handler quickly, but we cannot clear the status registers. 84 * 85 * To handle this, the system controller FPGA has the capability to connect the 86 * interrupt line to a user-controlled GPIO pin. This pin is driven high 87 * (unasserted) and left that way. To mask the interrupt, we change the 88 * interrupt source to the GPIO pin. Tada, we hid the interrupt. :) 89 */ 90 91#include <linux/of_platform.h> 92#include <linux/dma-mapping.h> 93#include <linux/miscdevice.h> 94#include <linux/interrupt.h> 95#include <linux/dmaengine.h> 96#include <linux/seq_file.h> 97#include <linux/highmem.h> 98#include <linux/debugfs.h> 99#include <linux/kernel.h> 100#include <linux/module.h> 101#include <linux/poll.h> 102#include <linux/init.h> 103#include <linux/slab.h> 104#include <linux/kref.h> 105#include <linux/io.h> 106 107#include <media/videobuf-dma-sg.h> 108 109/* system controller registers */ 110#define SYS_IRQ_SOURCE_CTL 0x24 111#define SYS_IRQ_OUTPUT_EN 0x28 112#define SYS_IRQ_OUTPUT_DATA 0x2C 113#define SYS_IRQ_INPUT_DATA 0x30 114#define SYS_FPGA_CONFIG_STATUS 0x44 115 116/* GPIO IRQ line assignment */ 117#define IRQ_CORL_DONE 0x10 118 119/* FPGA registers */ 120#define MMAP_REG_VERSION 0x00 121#define MMAP_REG_CORL_CONF1 0x08 122#define MMAP_REG_CORL_CONF2 0x0C 123#define MMAP_REG_STATUS 0x48 124 125#define SYS_FPGA_BLOCK 0xF0000000 126 127#define DATA_FPGA_START 0x400000 128#define DATA_FPGA_SIZE 0x80000 129 130static const char drv_name[] = "carma-fpga"; 131 132#define NUM_FPGA 4 133 134#define MIN_DATA_BUFS 8 135#define MAX_DATA_BUFS 64 136 137struct fpga_info { 138 unsigned int num_lag_ram; 139 unsigned int blk_size; 140}; 141 142struct data_buf { 143 struct list_head entry; 144 struct videobuf_dmabuf vb; 145 size_t size; 146}; 147 148struct fpga_device { 149 /* character device */ 150 struct miscdevice miscdev; 151 struct device *dev; 152 struct mutex mutex; 153 154 /* reference count */ 155 struct kref ref; 156 157 /* FPGA registers and information */ 158 struct fpga_info info[NUM_FPGA]; 159 void __iomem *regs; 160 int irq; 161 162 /* FPGA Physical Address/Size Information */ 163 resource_size_t phys_addr; 164 size_t phys_size; 165 166 /* DMA structures */ 167 struct sg_table corl_table; 168 unsigned int corl_nents; 169 struct dma_chan *chan; 170 171 /* Protection for all members below */ 172 spinlock_t lock; 173 174 /* Device enable/disable flag */ 175 bool enabled; 176 177 /* Correlation data buffers */ 178 wait_queue_head_t wait; 179 struct list_head free; 180 struct list_head used; 181 struct data_buf *inflight; 182 183 /* Information about data buffers */ 184 unsigned int num_dropped; 185 unsigned int num_buffers; 186 size_t bufsize; 187 struct dentry *dbg_entry; 188}; 189 190struct fpga_reader { 191 struct fpga_device *priv; 192 struct data_buf *buf; 193 off_t buf_start; 194}; 195 196static void fpga_device_release(struct kref *ref) 197{ 198 struct fpga_device *priv = container_of(ref, struct fpga_device, ref); 199 200 /* the last reader has exited, cleanup the last bits */ 201 mutex_destroy(&priv->mutex); 202 kfree(priv); 203} 204 205/* 206 * Data Buffer Allocation Helpers 207 */ 208 209/** 210 * data_free_buffer() - free a single data buffer and all allocated memory 211 * @buf: the buffer to free 212 * 213 * This will free all of the pages allocated to the given data buffer, and 214 * then free the structure itself 215 */ 216static void data_free_buffer(struct data_buf *buf) 217{ 218 /* It is ok to free a NULL buffer */ 219 if (!buf) 220 return; 221 222 /* free all memory */ 223 videobuf_dma_free(&buf->vb); 224 kfree(buf); 225} 226 227/** 228 * data_alloc_buffer() - allocate and fill a data buffer with pages 229 * @bytes: the number of bytes required 230 * 231 * This allocates all space needed for a data buffer. It must be mapped before 232 * use in a DMA transaction using videobuf_dma_map(). 233 * 234 * Returns NULL on failure 235 */ 236static struct data_buf *data_alloc_buffer(const size_t bytes) 237{ 238 unsigned int nr_pages; 239 struct data_buf *buf; 240 int ret; 241 242 /* calculate the number of pages necessary */ 243 nr_pages = DIV_ROUND_UP(bytes, PAGE_SIZE); 244 245 /* allocate the buffer structure */ 246 buf = kzalloc(sizeof(*buf), GFP_KERNEL); 247 if (!buf) 248 goto out_return; 249 250 /* initialize internal fields */ 251 INIT_LIST_HEAD(&buf->entry); 252 buf->size = bytes; 253 254 /* allocate the videobuf */ 255 videobuf_dma_init(&buf->vb); 256 ret = videobuf_dma_init_kernel(&buf->vb, DMA_FROM_DEVICE, nr_pages); 257 if (ret) 258 goto out_free_buf; 259 260 return buf; 261 262out_free_buf: 263 kfree(buf); 264out_return: 265 return NULL; 266} 267 268/** 269 * data_free_buffers() - free all allocated buffers 270 * @priv: the driver's private data structure 271 * 272 * Free all buffers allocated by the driver (except those currently in the 273 * process of being read by userspace). 274 * 275 * LOCKING: must hold dev->mutex 276 * CONTEXT: user 277 */ 278static void data_free_buffers(struct fpga_device *priv) 279{ 280 struct data_buf *buf, *tmp; 281 282 /* the device should be stopped, no DMA in progress */ 283 BUG_ON(priv->inflight != NULL); 284 285 list_for_each_entry_safe(buf, tmp, &priv->free, entry) { 286 list_del_init(&buf->entry); 287 videobuf_dma_unmap(priv->dev, &buf->vb); 288 data_free_buffer(buf); 289 } 290 291 list_for_each_entry_safe(buf, tmp, &priv->used, entry) { 292 list_del_init(&buf->entry); 293 videobuf_dma_unmap(priv->dev, &buf->vb); 294 data_free_buffer(buf); 295 } 296 297 priv->num_buffers = 0; 298 priv->bufsize = 0; 299} 300 301/** 302 * data_alloc_buffers() - allocate 1 seconds worth of data buffers 303 * @priv: the driver's private data structure 304 * 305 * Allocate enough buffers for a whole second worth of data 306 * 307 * This routine will attempt to degrade nicely by succeeding even if a full 308 * second worth of data buffers could not be allocated, as long as a minimum 309 * number were allocated. In this case, it will print a message to the kernel 310 * log. 311 * 312 * The device must not be modifying any lists when this is called. 313 * 314 * CONTEXT: user 315 * LOCKING: must hold dev->mutex 316 * 317 * Returns 0 on success, -ERRNO otherwise 318 */ 319static int data_alloc_buffers(struct fpga_device *priv) 320{ 321 struct data_buf *buf; 322 int i, ret; 323 324 for (i = 0; i < MAX_DATA_BUFS; i++) { 325 326 /* allocate a buffer */ 327 buf = data_alloc_buffer(priv->bufsize); 328 if (!buf) 329 break; 330 331 /* map it for DMA */ 332 ret = videobuf_dma_map(priv->dev, &buf->vb); 333 if (ret) { 334 data_free_buffer(buf); 335 break; 336 } 337 338 /* add it to the list of free buffers */ 339 list_add_tail(&buf->entry, &priv->free); 340 priv->num_buffers++; 341 } 342 343 /* Make sure we allocated the minimum required number of buffers */ 344 if (priv->num_buffers < MIN_DATA_BUFS) { 345 dev_err(priv->dev, "Unable to allocate enough data buffers\n"); 346 data_free_buffers(priv); 347 return -ENOMEM; 348 } 349 350 /* Warn if we are running in a degraded state, but do not fail */ 351 if (priv->num_buffers < MAX_DATA_BUFS) { 352 dev_warn(priv->dev, 353 "Unable to allocate %d buffers, using %d buffers instead\n", 354 MAX_DATA_BUFS, i); 355 } 356 357 return 0; 358} 359 360/* 361 * DMA Operations Helpers 362 */ 363 364/** 365 * fpga_start_addr() - get the physical address a DATA-FPGA 366 * @priv: the driver's private data structure 367 * @fpga: the DATA-FPGA number (zero based) 368 */ 369static dma_addr_t fpga_start_addr(struct fpga_device *priv, unsigned int fpga) 370{ 371 return priv->phys_addr + 0x400000 + (0x80000 * fpga); 372} 373 374/** 375 * fpga_block_addr() - get the physical address of a correlation data block 376 * @priv: the driver's private data structure 377 * @fpga: the DATA-FPGA number (zero based) 378 * @blknum: the correlation block number (zero based) 379 */ 380static dma_addr_t fpga_block_addr(struct fpga_device *priv, unsigned int fpga, 381 unsigned int blknum) 382{ 383 return fpga_start_addr(priv, fpga) + (0x10000 * (1 + blknum)); 384} 385 386#define REG_BLOCK_SIZE (32 * 4) 387 388/** 389 * data_setup_corl_table() - create the scatterlist for correlation dumps 390 * @priv: the driver's private data structure 391 * 392 * Create the scatterlist for transferring a correlation dump from the 393 * DATA FPGAs. This structure will be reused for each buffer than needs 394 * to be filled with correlation data. 395 * 396 * Returns 0 on success, -ERRNO otherwise 397 */ 398static int data_setup_corl_table(struct fpga_device *priv) 399{ 400 struct sg_table *table = &priv->corl_table; 401 struct scatterlist *sg; 402 struct fpga_info *info; 403 int i, j, ret; 404 405 /* Calculate the number of entries needed */ 406 priv->corl_nents = (1 + NUM_FPGA) * REG_BLOCK_SIZE; 407 for (i = 0; i < NUM_FPGA; i++) 408 priv->corl_nents += priv->info[i].num_lag_ram; 409 410 /* Allocate the scatterlist table */ 411 ret = sg_alloc_table(table, priv->corl_nents, GFP_KERNEL); 412 if (ret) { 413 dev_err(priv->dev, "unable to allocate DMA table\n"); 414 return ret; 415 } 416 417 /* Add the DATA FPGA registers to the scatterlist */ 418 sg = table->sgl; 419 for (i = 0; i < NUM_FPGA; i++) { 420 sg_dma_address(sg) = fpga_start_addr(priv, i); 421 sg_dma_len(sg) = REG_BLOCK_SIZE; 422 sg = sg_next(sg); 423 } 424 425 /* Add the SYS-FPGA registers to the scatterlist */ 426 sg_dma_address(sg) = SYS_FPGA_BLOCK; 427 sg_dma_len(sg) = REG_BLOCK_SIZE; 428 sg = sg_next(sg); 429 430 /* Add the FPGA correlation data blocks to the scatterlist */ 431 for (i = 0; i < NUM_FPGA; i++) { 432 info = &priv->info[i]; 433 for (j = 0; j < info->num_lag_ram; j++) { 434 sg_dma_address(sg) = fpga_block_addr(priv, i, j); 435 sg_dma_len(sg) = info->blk_size; 436 sg = sg_next(sg); 437 } 438 } 439 440 /* 441 * All physical addresses and lengths are present in the structure 442 * now. It can be reused for every FPGA DATA interrupt 443 */ 444 return 0; 445} 446 447/* 448 * FPGA Register Access Helpers 449 */ 450 451static void fpga_write_reg(struct fpga_device *priv, unsigned int fpga, 452 unsigned int reg, u32 val) 453{ 454 const int fpga_start = DATA_FPGA_START + (fpga * DATA_FPGA_SIZE); 455 iowrite32be(val, priv->regs + fpga_start + reg); 456} 457 458static u32 fpga_read_reg(struct fpga_device *priv, unsigned int fpga, 459 unsigned int reg) 460{ 461 const int fpga_start = DATA_FPGA_START + (fpga * DATA_FPGA_SIZE); 462 return ioread32be(priv->regs + fpga_start + reg); 463} 464 465/** 466 * data_calculate_bufsize() - calculate the data buffer size required 467 * @priv: the driver's private data structure 468 * 469 * Calculate the total buffer size needed to hold a single block 470 * of correlation data 471 * 472 * CONTEXT: user 473 * 474 * Returns 0 on success, -ERRNO otherwise 475 */ 476static int data_calculate_bufsize(struct fpga_device *priv) 477{ 478 u32 num_corl, num_lags, num_meta, num_qcnt, num_pack; 479 u32 conf1, conf2, version; 480 u32 num_lag_ram, blk_size; 481 int i; 482 483 /* Each buffer starts with the 5 FPGA register areas */ 484 priv->bufsize = (1 + NUM_FPGA) * REG_BLOCK_SIZE; 485 486 /* Read and store the configuration data for each FPGA */ 487 for (i = 0; i < NUM_FPGA; i++) { 488 version = fpga_read_reg(priv, i, MMAP_REG_VERSION); 489 conf1 = fpga_read_reg(priv, i, MMAP_REG_CORL_CONF1); 490 conf2 = fpga_read_reg(priv, i, MMAP_REG_CORL_CONF2); 491 492 /* minor version 2 and later */ 493 if ((version & 0x000000FF) >= 2) { 494 num_corl = (conf1 & 0x000000F0) >> 4; 495 num_pack = (conf1 & 0x00000F00) >> 8; 496 num_lags = (conf1 & 0x00FFF000) >> 12; 497 num_meta = (conf1 & 0x7F000000) >> 24; 498 num_qcnt = (conf2 & 0x00000FFF) >> 0; 499 } else { 500 num_corl = (conf1 & 0x000000F0) >> 4; 501 num_pack = 1; /* implied */ 502 num_lags = (conf1 & 0x000FFF00) >> 8; 503 num_meta = (conf1 & 0x7FF00000) >> 20; 504 num_qcnt = (conf2 & 0x00000FFF) >> 0; 505 } 506 507 num_lag_ram = (num_corl + num_pack - 1) / num_pack; 508 blk_size = ((num_pack * num_lags) + num_meta + num_qcnt) * 8; 509 510 priv->info[i].num_lag_ram = num_lag_ram; 511 priv->info[i].blk_size = blk_size; 512 priv->bufsize += num_lag_ram * blk_size; 513 514 dev_dbg(priv->dev, "FPGA %d NUM_CORL: %d\n", i, num_corl); 515 dev_dbg(priv->dev, "FPGA %d NUM_PACK: %d\n", i, num_pack); 516 dev_dbg(priv->dev, "FPGA %d NUM_LAGS: %d\n", i, num_lags); 517 dev_dbg(priv->dev, "FPGA %d NUM_META: %d\n", i, num_meta); 518 dev_dbg(priv->dev, "FPGA %d NUM_QCNT: %d\n", i, num_qcnt); 519 dev_dbg(priv->dev, "FPGA %d BLK_SIZE: %d\n", i, blk_size); 520 } 521 522 dev_dbg(priv->dev, "TOTAL BUFFER SIZE: %zu bytes\n", priv->bufsize); 523 return 0; 524} 525 526/* 527 * Interrupt Handling 528 */ 529 530/** 531 * data_disable_interrupts() - stop the device from generating interrupts 532 * @priv: the driver's private data structure 533 * 534 * Hide interrupts by switching to GPIO interrupt source 535 * 536 * LOCKING: must hold dev->lock 537 */ 538static void data_disable_interrupts(struct fpga_device *priv) 539{ 540 /* hide the interrupt by switching the IRQ driver to GPIO */ 541 iowrite32be(0x2F, priv->regs + SYS_IRQ_SOURCE_CTL); 542} 543 544/** 545 * data_enable_interrupts() - allow the device to generate interrupts 546 * @priv: the driver's private data structure 547 * 548 * Unhide interrupts by switching to the FPGA interrupt source. At the 549 * same time, clear the DATA-FPGA status registers. 550 * 551 * LOCKING: must hold dev->lock 552 */ 553static void data_enable_interrupts(struct fpga_device *priv) 554{ 555 /* clear the actual FPGA corl_done interrupt */ 556 fpga_write_reg(priv, 0, MMAP_REG_STATUS, 0x0); 557 fpga_write_reg(priv, 1, MMAP_REG_STATUS, 0x0); 558 fpga_write_reg(priv, 2, MMAP_REG_STATUS, 0x0); 559 fpga_write_reg(priv, 3, MMAP_REG_STATUS, 0x0); 560 561 /* flush the writes */ 562 fpga_read_reg(priv, 0, MMAP_REG_STATUS); 563 fpga_read_reg(priv, 1, MMAP_REG_STATUS); 564 fpga_read_reg(priv, 2, MMAP_REG_STATUS); 565 fpga_read_reg(priv, 3, MMAP_REG_STATUS); 566 567 /* switch back to the external interrupt source */ 568 iowrite32be(0x3F, priv->regs + SYS_IRQ_SOURCE_CTL); 569} 570 571/** 572 * data_dma_cb() - DMAEngine callback for DMA completion 573 * @data: the driver's private data structure 574 * 575 * Complete a DMA transfer from the DATA-FPGA's 576 * 577 * This is called via the DMA callback mechanism, and will handle moving the 578 * completed DMA transaction to the used list, and then wake any processes 579 * waiting for new data 580 * 581 * CONTEXT: any, softirq expected 582 */ 583static void data_dma_cb(void *data) 584{ 585 struct fpga_device *priv = data; 586 unsigned long flags; 587 588 spin_lock_irqsave(&priv->lock, flags); 589 590 /* If there is no inflight buffer, we've got a bug */ 591 BUG_ON(priv->inflight == NULL); 592 593 /* Move the inflight buffer onto the used list */ 594 list_move_tail(&priv->inflight->entry, &priv->used); 595 priv->inflight = NULL; 596 597 /* 598 * If data dumping is still enabled, then clear the FPGA 599 * status registers and re-enable FPGA interrupts 600 */ 601 if (priv->enabled) 602 data_enable_interrupts(priv); 603 604 spin_unlock_irqrestore(&priv->lock, flags); 605 606 /* 607 * We've changed both the inflight and used lists, so we need 608 * to wake up any processes that are blocking for those events 609 */ 610 wake_up(&priv->wait); 611} 612 613/** 614 * data_submit_dma() - prepare and submit the required DMA to fill a buffer 615 * @priv: the driver's private data structure 616 * @buf: the data buffer 617 * 618 * Prepare and submit the necessary DMA transactions to fill a correlation 619 * data buffer. 620 * 621 * LOCKING: must hold dev->lock 622 * CONTEXT: hardirq only 623 * 624 * Returns 0 on success, -ERRNO otherwise 625 */ 626static int data_submit_dma(struct fpga_device *priv, struct data_buf *buf) 627{ 628 struct scatterlist *dst_sg, *src_sg; 629 unsigned int dst_nents, src_nents; 630 struct dma_chan *chan = priv->chan; 631 struct dma_async_tx_descriptor *tx; 632 dma_cookie_t cookie; 633 dma_addr_t dst, src; 634 635 dst_sg = buf->vb.sglist; 636 dst_nents = buf->vb.sglen; 637 638 src_sg = priv->corl_table.sgl; 639 src_nents = priv->corl_nents; 640 641 /* 642 * All buffers passed to this function should be ready and mapped 643 * for DMA already. Therefore, we don't need to do anything except 644 * submit it to the Freescale DMA Engine for processing 645 */ 646 647 /* setup the scatterlist to scatterlist transfer */ 648 tx = chan->device->device_prep_dma_sg(chan, 649 dst_sg, dst_nents, 650 src_sg, src_nents, 651 0); 652 if (!tx) { 653 dev_err(priv->dev, "unable to prep scatterlist DMA\n"); 654 return -ENOMEM; 655 } 656 657 /* submit the transaction to the DMA controller */ 658 cookie = tx->tx_submit(tx); 659 if (dma_submit_error(cookie)) { 660 dev_err(priv->dev, "unable to submit scatterlist DMA\n"); 661 return -ENOMEM; 662 } 663 664 /* Prepare the re-read of the SYS-FPGA block */ 665 dst = sg_dma_address(dst_sg) + (NUM_FPGA * REG_BLOCK_SIZE); 666 src = SYS_FPGA_BLOCK; 667 tx = chan->device->device_prep_dma_memcpy(chan, dst, src, 668 REG_BLOCK_SIZE, 669 DMA_PREP_INTERRUPT); 670 if (!tx) { 671 dev_err(priv->dev, "unable to prep SYS-FPGA DMA\n"); 672 return -ENOMEM; 673 } 674 675 /* Setup the callback */ 676 tx->callback = data_dma_cb; 677 tx->callback_param = priv; 678 679 /* submit the transaction to the DMA controller */ 680 cookie = tx->tx_submit(tx); 681 if (dma_submit_error(cookie)) { 682 dev_err(priv->dev, "unable to submit SYS-FPGA DMA\n"); 683 return -ENOMEM; 684 } 685 686 return 0; 687} 688 689#define CORL_DONE 0x1 690#define CORL_ERR 0x2 691 692static irqreturn_t data_irq(int irq, void *dev_id) 693{ 694 struct fpga_device *priv = dev_id; 695 bool submitted = false; 696 struct data_buf *buf; 697 u32 status; 698 int i; 699 700 /* detect spurious interrupts via FPGA status */ 701 for (i = 0; i < 4; i++) { 702 status = fpga_read_reg(priv, i, MMAP_REG_STATUS); 703 if (!(status & (CORL_DONE | CORL_ERR))) { 704 dev_err(priv->dev, "spurious irq detected (FPGA)\n"); 705 return IRQ_NONE; 706 } 707 } 708 709 /* detect spurious interrupts via raw IRQ pin readback */ 710 status = ioread32be(priv->regs + SYS_IRQ_INPUT_DATA); 711 if (status & IRQ_CORL_DONE) { 712 dev_err(priv->dev, "spurious irq detected (IRQ)\n"); 713 return IRQ_NONE; 714 } 715 716 spin_lock(&priv->lock); 717 718 /* 719 * This is an error case that should never happen. 720 * 721 * If this driver has a bug and manages to re-enable interrupts while 722 * a DMA is in progress, then we will hit this statement and should 723 * start paying attention immediately. 724 */ 725 BUG_ON(priv->inflight != NULL); 726 727 /* hide the interrupt by switching the IRQ driver to GPIO */ 728 data_disable_interrupts(priv); 729 730 /* If there are no free buffers, drop this data */ 731 if (list_empty(&priv->free)) { 732 priv->num_dropped++; 733 goto out; 734 } 735 736 buf = list_first_entry(&priv->free, struct data_buf, entry); 737 list_del_init(&buf->entry); 738 BUG_ON(buf->size != priv->bufsize); 739 740 /* Submit a DMA transfer to get the correlation data */ 741 if (data_submit_dma(priv, buf)) { 742 dev_err(priv->dev, "Unable to setup DMA transfer\n"); 743 list_move_tail(&buf->entry, &priv->free); 744 goto out; 745 } 746 747 /* Save the buffer for the DMA callback */ 748 priv->inflight = buf; 749 submitted = true; 750 751 /* Start the DMA Engine */ 752 dma_async_memcpy_issue_pending(priv->chan); 753 754out: 755 /* If no DMA was submitted, re-enable interrupts */ 756 if (!submitted) 757 data_enable_interrupts(priv); 758 759 spin_unlock(&priv->lock); 760 return IRQ_HANDLED; 761} 762 763/* 764 * Realtime Device Enable Helpers 765 */ 766 767/** 768 * data_device_enable() - enable the device for buffered dumping 769 * @priv: the driver's private data structure 770 * 771 * Enable the device for buffered dumping. Allocates buffers and hooks up 772 * the interrupt handler. When this finishes, data will come pouring in. 773 * 774 * LOCKING: must hold dev->mutex 775 * CONTEXT: user context only 776 * 777 * Returns 0 on success, -ERRNO otherwise 778 */ 779static int data_device_enable(struct fpga_device *priv) 780{ 781 bool enabled; 782 u32 val; 783 int ret; 784 785 /* multiple enables are safe: they do nothing */ 786 spin_lock_irq(&priv->lock); 787 enabled = priv->enabled; 788 spin_unlock_irq(&priv->lock); 789 if (enabled) 790 return 0; 791 792 /* check that the FPGAs are programmed */ 793 val = ioread32be(priv->regs + SYS_FPGA_CONFIG_STATUS); 794 if (!(val & (1 << 18))) { 795 dev_err(priv->dev, "DATA-FPGAs are not enabled\n"); 796 return -ENODATA; 797 } 798 799 /* read the FPGAs to calculate the buffer size */ 800 ret = data_calculate_bufsize(priv); 801 if (ret) { 802 dev_err(priv->dev, "unable to calculate buffer size\n"); 803 goto out_error; 804 } 805 806 /* allocate the correlation data buffers */ 807 ret = data_alloc_buffers(priv); 808 if (ret) { 809 dev_err(priv->dev, "unable to allocate buffers\n"); 810 goto out_error; 811 } 812 813 /* setup the source scatterlist for dumping correlation data */ 814 ret = data_setup_corl_table(priv); 815 if (ret) { 816 dev_err(priv->dev, "unable to setup correlation DMA table\n"); 817 goto out_error; 818 } 819 820 /* prevent the FPGAs from generating interrupts */ 821 data_disable_interrupts(priv); 822 823 /* hookup the irq handler */ 824 ret = request_irq(priv->irq, data_irq, IRQF_SHARED, drv_name, priv); 825 if (ret) { 826 dev_err(priv->dev, "unable to request IRQ handler\n"); 827 goto out_error; 828 } 829 830 /* allow the DMA callback to re-enable FPGA interrupts */ 831 spin_lock_irq(&priv->lock); 832 priv->enabled = true; 833 spin_unlock_irq(&priv->lock); 834 835 /* allow the FPGAs to generate interrupts */ 836 data_enable_interrupts(priv); 837 return 0; 838 839out_error: 840 sg_free_table(&priv->corl_table); 841 priv->corl_nents = 0; 842 843 data_free_buffers(priv); 844 return ret; 845} 846 847/** 848 * data_device_disable() - disable the device for buffered dumping 849 * @priv: the driver's private data structure 850 * 851 * Disable the device for buffered dumping. Stops new DMA transactions from 852 * being generated, waits for all outstanding DMA to complete, and then frees 853 * all buffers. 854 * 855 * LOCKING: must hold dev->mutex 856 * CONTEXT: user only 857 * 858 * Returns 0 on success, -ERRNO otherwise 859 */ 860static int data_device_disable(struct fpga_device *priv) 861{ 862 spin_lock_irq(&priv->lock); 863 864 /* allow multiple disable */ 865 if (!priv->enabled) { 866 spin_unlock_irq(&priv->lock); 867 return 0; 868 } 869 870 /* 871 * Mark the device disabled 872 * 873 * This stops DMA callbacks from re-enabling interrupts 874 */ 875 priv->enabled = false; 876 877 /* prevent the FPGAs from generating interrupts */ 878 data_disable_interrupts(priv); 879 880 /* wait until all ongoing DMA has finished */ 881 while (priv->inflight != NULL) { 882 spin_unlock_irq(&priv->lock); 883 wait_event(priv->wait, priv->inflight == NULL); 884 spin_lock_irq(&priv->lock); 885 } 886 887 spin_unlock_irq(&priv->lock); 888 889 /* unhook the irq handler */ 890 free_irq(priv->irq, priv); 891 892 /* free the correlation table */ 893 sg_free_table(&priv->corl_table); 894 priv->corl_nents = 0; 895 896 /* free all buffers: the free and used lists are not being changed */ 897 data_free_buffers(priv); 898 return 0; 899} 900 901/* 902 * DEBUGFS Interface 903 */ 904#ifdef CONFIG_DEBUG_FS 905 906/* 907 * Count the number of entries in the given list 908 */ 909static unsigned int list_num_entries(struct list_head *list) 910{ 911 struct list_head *entry; 912 unsigned int ret = 0; 913 914 list_for_each(entry, list) 915 ret++; 916 917 return ret; 918} 919 920static int data_debug_show(struct seq_file *f, void *offset) 921{ 922 struct fpga_device *priv = f->private; 923 924 spin_lock_irq(&priv->lock); 925 926 seq_printf(f, "enabled: %d\n", priv->enabled); 927 seq_printf(f, "bufsize: %d\n", priv->bufsize); 928 seq_printf(f, "num_buffers: %d\n", priv->num_buffers); 929 seq_printf(f, "num_free: %d\n", list_num_entries(&priv->free)); 930 seq_printf(f, "inflight: %d\n", priv->inflight != NULL); 931 seq_printf(f, "num_used: %d\n", list_num_entries(&priv->used)); 932 seq_printf(f, "num_dropped: %d\n", priv->num_dropped); 933 934 spin_unlock_irq(&priv->lock); 935 return 0; 936} 937 938static int data_debug_open(struct inode *inode, struct file *file) 939{ 940 return single_open(file, data_debug_show, inode->i_private); 941} 942 943static const struct file_operations data_debug_fops = { 944 .owner = THIS_MODULE, 945 .open = data_debug_open, 946 .read = seq_read, 947 .llseek = seq_lseek, 948 .release = single_release, 949}; 950 951static int data_debugfs_init(struct fpga_device *priv) 952{ 953 priv->dbg_entry = debugfs_create_file(drv_name, S_IRUGO, NULL, priv, 954 &data_debug_fops); 955 if (IS_ERR(priv->dbg_entry)) 956 return PTR_ERR(priv->dbg_entry); 957 958 return 0; 959} 960 961static void data_debugfs_exit(struct fpga_device *priv) 962{ 963 debugfs_remove(priv->dbg_entry); 964} 965 966#else 967 968static inline int data_debugfs_init(struct fpga_device *priv) 969{ 970 return 0; 971} 972 973static inline void data_debugfs_exit(struct fpga_device *priv) 974{ 975} 976 977#endif /* CONFIG_DEBUG_FS */ 978 979/* 980 * SYSFS Attributes 981 */ 982 983static ssize_t data_en_show(struct device *dev, struct device_attribute *attr, 984 char *buf) 985{ 986 struct fpga_device *priv = dev_get_drvdata(dev); 987 int ret; 988 989 spin_lock_irq(&priv->lock); 990 ret = snprintf(buf, PAGE_SIZE, "%u\n", priv->enabled); 991 spin_unlock_irq(&priv->lock); 992 993 return ret; 994} 995 996static ssize_t data_en_set(struct device *dev, struct device_attribute *attr, 997 const char *buf, size_t count) 998{ 999 struct fpga_device *priv = dev_get_drvdata(dev); 1000 unsigned long enable; 1001 int ret; 1002 1003 ret = strict_strtoul(buf, 0, &enable); 1004 if (ret) { 1005 dev_err(priv->dev, "unable to parse enable input\n"); 1006 return -EINVAL; 1007 } 1008 1009 /* protect against concurrent enable/disable */ 1010 ret = mutex_lock_interruptible(&priv->mutex); 1011 if (ret) 1012 return ret; 1013 1014 if (enable) 1015 ret = data_device_enable(priv); 1016 else 1017 ret = data_device_disable(priv); 1018 1019 if (ret) { 1020 dev_err(priv->dev, "device %s failed\n", 1021 enable ? "enable" : "disable"); 1022 count = ret; 1023 goto out_unlock; 1024 } 1025 1026out_unlock: 1027 mutex_unlock(&priv->mutex); 1028 return count; 1029} 1030 1031static DEVICE_ATTR(enable, S_IWUSR | S_IRUGO, data_en_show, data_en_set); 1032 1033static struct attribute *data_sysfs_attrs[] = { 1034 &dev_attr_enable.attr, 1035 NULL, 1036}; 1037 1038static const struct attribute_group rt_sysfs_attr_group = { 1039 .attrs = data_sysfs_attrs, 1040}; 1041 1042/* 1043 * FPGA Realtime Data Character Device 1044 */ 1045 1046static int data_open(struct inode *inode, struct file *filp) 1047{ 1048 /* 1049 * The miscdevice layer puts our struct miscdevice into the 1050 * filp->private_data field. We use this to find our private 1051 * data and then overwrite it with our own private structure. 1052 */ 1053 struct fpga_device *priv = container_of(filp->private_data, 1054 struct fpga_device, miscdev); 1055 struct fpga_reader *reader; 1056 int ret; 1057 1058 /* allocate private data */ 1059 reader = kzalloc(sizeof(*reader), GFP_KERNEL); 1060 if (!reader) 1061 return -ENOMEM; 1062 1063 reader->priv = priv; 1064 reader->buf = NULL; 1065 1066 filp->private_data = reader; 1067 ret = nonseekable_open(inode, filp); 1068 if (ret) { 1069 dev_err(priv->dev, "nonseekable-open failed\n"); 1070 kfree(reader); 1071 return ret; 1072 } 1073 1074 /* 1075 * success, increase the reference count of the private data structure 1076 * so that it doesn't disappear if the device is unbound 1077 */ 1078 kref_get(&priv->ref); 1079 return 0; 1080} 1081 1082static int data_release(struct inode *inode, struct file *filp) 1083{ 1084 struct fpga_reader *reader = filp->private_data; 1085 struct fpga_device *priv = reader->priv; 1086 1087 /* free the per-reader structure */ 1088 data_free_buffer(reader->buf); 1089 kfree(reader); 1090 filp->private_data = NULL; 1091 1092 /* decrement our reference count to the private data */ 1093 kref_put(&priv->ref, fpga_device_release); 1094 return 0; 1095} 1096 1097static ssize_t data_read(struct file *filp, char __user *ubuf, size_t count, 1098 loff_t *f_pos) 1099{ 1100 struct fpga_reader *reader = filp->private_data; 1101 struct fpga_device *priv = reader->priv; 1102 struct list_head *used = &priv->used; 1103 bool drop_buffer = false; 1104 struct data_buf *dbuf; 1105 size_t avail; 1106 void *data; 1107 int ret; 1108 1109 /* check if we already have a partial buffer */ 1110 if (reader->buf) { 1111 dbuf = reader->buf; 1112 goto have_buffer; 1113 } 1114 1115 spin_lock_irq(&priv->lock); 1116 1117 /* Block until there is at least one buffer on the used list */ 1118 while (list_empty(used)) { 1119 spin_unlock_irq(&priv->lock); 1120 1121 if (filp->f_flags & O_NONBLOCK) 1122 return -EAGAIN; 1123 1124 ret = wait_event_interruptible(priv->wait, !list_empty(used)); 1125 if (ret) 1126 return ret; 1127 1128 spin_lock_irq(&priv->lock); 1129 } 1130 1131 /* Grab the first buffer off of the used list */ 1132 dbuf = list_first_entry(used, struct data_buf, entry); 1133 list_del_init(&dbuf->entry); 1134 1135 spin_unlock_irq(&priv->lock); 1136 1137 /* Buffers are always mapped: unmap it */ 1138 videobuf_dma_unmap(priv->dev, &dbuf->vb); 1139 1140 /* save the buffer for later */ 1141 reader->buf = dbuf; 1142 reader->buf_start = 0; 1143 1144have_buffer: 1145 /* Get the number of bytes available */ 1146 avail = dbuf->size - reader->buf_start; 1147 data = dbuf->vb.vaddr + reader->buf_start; 1148 1149 /* Get the number of bytes we can transfer */ 1150 count = min(count, avail); 1151 1152 /* Copy the data to the userspace buffer */ 1153 if (copy_to_user(ubuf, data, count)) 1154 return -EFAULT; 1155 1156 /* Update the amount of available space */ 1157 avail -= count; 1158 1159 /* 1160 * If there is still some data available, save the buffer for the 1161 * next userspace call to read() and return 1162 */ 1163 if (avail > 0) { 1164 reader->buf_start += count; 1165 reader->buf = dbuf; 1166 return count; 1167 } 1168 1169 /* 1170 * Get the buffer ready to be reused for DMA 1171 * 1172 * If it fails, we pretend that the read never happed and return 1173 * -EFAULT to userspace. The read will be retried. 1174 */ 1175 ret = videobuf_dma_map(priv->dev, &dbuf->vb); 1176 if (ret) { 1177 dev_err(priv->dev, "unable to remap buffer for DMA\n"); 1178 return -EFAULT; 1179 } 1180 1181 /* Lock against concurrent enable/disable */ 1182 spin_lock_irq(&priv->lock); 1183 1184 /* the reader is finished with this buffer */ 1185 reader->buf = NULL; 1186 1187 /* 1188 * One of two things has happened, the device is disabled, or the 1189 * device has been reconfigured underneath us. In either case, we 1190 * should just throw away the buffer. 1191 * 1192 * Lockdep complains if this is done under the spinlock, so we 1193 * handle it during the unlock path. 1194 */ 1195 if (!priv->enabled || dbuf->size != priv->bufsize) { 1196 drop_buffer = true; 1197 goto out_unlock; 1198 } 1199 1200 /* The buffer is safe to reuse, so add it back to the free list */ 1201 list_add_tail(&dbuf->entry, &priv->free); 1202 1203out_unlock: 1204 spin_unlock_irq(&priv->lock); 1205 1206 if (drop_buffer) { 1207 videobuf_dma_unmap(priv->dev, &dbuf->vb); 1208 data_free_buffer(dbuf); 1209 } 1210 1211 return count; 1212} 1213 1214static unsigned int data_poll(struct file *filp, struct poll_table_struct *tbl) 1215{ 1216 struct fpga_reader *reader = filp->private_data; 1217 struct fpga_device *priv = reader->priv; 1218 unsigned int mask = 0; 1219 1220 poll_wait(filp, &priv->wait, tbl); 1221 1222 if (!list_empty(&priv->used)) 1223 mask |= POLLIN | POLLRDNORM; 1224 1225 return mask; 1226} 1227 1228static int data_mmap(struct file *filp, struct vm_area_struct *vma) 1229{ 1230 struct fpga_reader *reader = filp->private_data; 1231 struct fpga_device *priv = reader->priv; 1232 unsigned long offset, vsize, psize, addr; 1233 1234 /* VMA properties */ 1235 offset = vma->vm_pgoff << PAGE_SHIFT; 1236 vsize = vma->vm_end - vma->vm_start; 1237 psize = priv->phys_size - offset; 1238 addr = (priv->phys_addr + offset) >> PAGE_SHIFT; 1239 1240 /* Check against the FPGA region's physical memory size */ 1241 if (vsize > psize) { 1242 dev_err(priv->dev, "requested mmap mapping too large\n"); 1243 return -EINVAL; 1244 } 1245 1246 /* IO memory (stop cacheing) */ 1247 vma->vm_flags |= VM_IO | VM_RESERVED; 1248 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); 1249 1250 return io_remap_pfn_range(vma, vma->vm_start, addr, vsize, 1251 vma->vm_page_prot); 1252} 1253 1254static const struct file_operations data_fops = { 1255 .owner = THIS_MODULE, 1256 .open = data_open, 1257 .release = data_release, 1258 .read = data_read, 1259 .poll = data_poll, 1260 .mmap = data_mmap, 1261 .llseek = no_llseek, 1262}; 1263 1264/* 1265 * OpenFirmware Device Subsystem 1266 */ 1267 1268static bool dma_filter(struct dma_chan *chan, void *data) 1269{ 1270 /* 1271 * DMA Channel #0 is used for the FPGA Programmer, so ignore it 1272 * 1273 * This probably won't survive an unload/load cycle of the Freescale 1274 * DMAEngine driver, but that won't be a problem 1275 */ 1276 if (chan->chan_id == 0 && chan->device->dev_id == 0) 1277 return false; 1278 1279 return true; 1280} 1281 1282static int data_of_probe(struct platform_device *op) 1283{ 1284 struct device_node *of_node = op->dev.of_node; 1285 struct device *this_device; 1286 struct fpga_device *priv; 1287 struct resource res; 1288 dma_cap_mask_t mask; 1289 int ret; 1290 1291 /* Allocate private data */ 1292 priv = kzalloc(sizeof(*priv), GFP_KERNEL); 1293 if (!priv) { 1294 dev_err(&op->dev, "Unable to allocate device private data\n"); 1295 ret = -ENOMEM; 1296 goto out_return; 1297 } 1298 1299 dev_set_drvdata(&op->dev, priv); 1300 priv->dev = &op->dev; 1301 kref_init(&priv->ref); 1302 mutex_init(&priv->mutex); 1303 1304 dev_set_drvdata(priv->dev, priv); 1305 spin_lock_init(&priv->lock); 1306 INIT_LIST_HEAD(&priv->free); 1307 INIT_LIST_HEAD(&priv->used); 1308 init_waitqueue_head(&priv->wait); 1309 1310 /* Setup the misc device */ 1311 priv->miscdev.minor = MISC_DYNAMIC_MINOR; 1312 priv->miscdev.name = drv_name; 1313 priv->miscdev.fops = &data_fops; 1314 1315 /* Get the physical address of the FPGA registers */ 1316 ret = of_address_to_resource(of_node, 0, &res); 1317 if (ret) { 1318 dev_err(&op->dev, "Unable to find FPGA physical address\n"); 1319 ret = -ENODEV; 1320 goto out_free_priv; 1321 } 1322 1323 priv->phys_addr = res.start; 1324 priv->phys_size = resource_size(&res); 1325 1326 /* ioremap the registers for use */ 1327 priv->regs = of_iomap(of_node, 0); 1328 if (!priv->regs) { 1329 dev_err(&op->dev, "Unable to ioremap registers\n"); 1330 ret = -ENOMEM; 1331 goto out_free_priv; 1332 } 1333 1334 dma_cap_zero(mask); 1335 dma_cap_set(DMA_MEMCPY, mask); 1336 dma_cap_set(DMA_INTERRUPT, mask); 1337 dma_cap_set(DMA_SLAVE, mask); 1338 dma_cap_set(DMA_SG, mask); 1339 1340 /* Request a DMA channel */ 1341 priv->chan = dma_request_channel(mask, dma_filter, NULL); 1342 if (!priv->chan) { 1343 dev_err(&op->dev, "Unable to request DMA channel\n"); 1344 ret = -ENODEV; 1345 goto out_unmap_regs; 1346 } 1347 1348 /* Find the correct IRQ number */ 1349 priv->irq = irq_of_parse_and_map(of_node, 0); 1350 if (priv->irq == NO_IRQ) { 1351 dev_err(&op->dev, "Unable to find IRQ line\n"); 1352 ret = -ENODEV; 1353 goto out_release_dma; 1354 } 1355 1356 /* Drive the GPIO for FPGA IRQ high (no interrupt) */ 1357 iowrite32be(IRQ_CORL_DONE, priv->regs + SYS_IRQ_OUTPUT_DATA); 1358 1359 /* Register the miscdevice */ 1360 ret = misc_register(&priv->miscdev); 1361 if (ret) { 1362 dev_err(&op->dev, "Unable to register miscdevice\n"); 1363 goto out_irq_dispose_mapping; 1364 } 1365 1366 /* Create the debugfs files */ 1367 ret = data_debugfs_init(priv); 1368 if (ret) { 1369 dev_err(&op->dev, "Unable to create debugfs files\n"); 1370 goto out_misc_deregister; 1371 } 1372 1373 /* Create the sysfs files */ 1374 this_device = priv->miscdev.this_device; 1375 dev_set_drvdata(this_device, priv); 1376 ret = sysfs_create_group(&this_device->kobj, &rt_sysfs_attr_group); 1377 if (ret) { 1378 dev_err(&op->dev, "Unable to create sysfs files\n"); 1379 goto out_data_debugfs_exit; 1380 } 1381 1382 dev_info(&op->dev, "CARMA FPGA Realtime Data Driver Loaded\n"); 1383 return 0; 1384 1385out_data_debugfs_exit: 1386 data_debugfs_exit(priv); 1387out_misc_deregister: 1388 misc_deregister(&priv->miscdev); 1389out_irq_dispose_mapping: 1390 irq_dispose_mapping(priv->irq); 1391out_release_dma: 1392 dma_release_channel(priv->chan); 1393out_unmap_regs: 1394 iounmap(priv->regs); 1395out_free_priv: 1396 kref_put(&priv->ref, fpga_device_release); 1397out_return: 1398 return ret; 1399} 1400 1401static int data_of_remove(struct platform_device *op) 1402{ 1403 struct fpga_device *priv = dev_get_drvdata(&op->dev); 1404 struct device *this_device = priv->miscdev.this_device; 1405 1406 /* remove all sysfs files, now the device cannot be re-enabled */ 1407 sysfs_remove_group(&this_device->kobj, &rt_sysfs_attr_group); 1408 1409 /* remove all debugfs files */ 1410 data_debugfs_exit(priv); 1411 1412 /* disable the device from generating data */ 1413 data_device_disable(priv); 1414 1415 /* remove the character device to stop new readers from appearing */ 1416 misc_deregister(&priv->miscdev); 1417 1418 /* cleanup everything not needed by readers */ 1419 irq_dispose_mapping(priv->irq); 1420 dma_release_channel(priv->chan); 1421 iounmap(priv->regs); 1422 1423 /* release our reference */ 1424 kref_put(&priv->ref, fpga_device_release); 1425 return 0; 1426} 1427 1428static struct of_device_id data_of_match[] = { 1429 { .compatible = "carma,carma-fpga", }, 1430 {}, 1431}; 1432 1433static struct platform_driver data_of_driver = { 1434 .probe = data_of_probe, 1435 .remove = data_of_remove, 1436 .driver = { 1437 .name = drv_name, 1438 .of_match_table = data_of_match, 1439 .owner = THIS_MODULE, 1440 }, 1441}; 1442 1443module_platform_driver(data_of_driver); 1444 1445MODULE_AUTHOR("Ira W. Snyder <iws@ovro.caltech.edu>"); 1446MODULE_DESCRIPTION("CARMA DATA-FPGA Access Driver"); 1447MODULE_LICENSE("GPL"); 1448