1/* 2 * Intel MIC Platform Software Stack (MPSS) 3 * 4 * Copyright(c) 2013 Intel Corporation. 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License, version 2, as 8 * published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope that it will be useful, but 11 * WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 * General Public License for more details. 14 * 15 * The full GNU General Public License is included in this distribution in 16 * the file called "COPYING". 17 * 18 * Disclaimer: The codes contained in these modules may be specific to 19 * the Intel Software Development Platform codenamed: Knights Ferry, and 20 * the Intel product codenamed: Knights Corner, and are not backward 21 * compatible with other Intel products. Additionally, Intel will NOT 22 * support the codes or instruction set in future products. 23 * 24 * Adapted from: 25 * 26 * virtio for kvm on s390 27 * 28 * Copyright IBM Corp. 2008 29 * 30 * This program is free software; you can redistribute it and/or modify 31 * it under the terms of the GNU General Public License (version 2 only) 32 * as published by the Free Software Foundation. 33 * 34 * Author(s): Christian Borntraeger <borntraeger@de.ibm.com> 35 * 36 * Intel MIC Card driver. 37 * 38 */ 39#include <linux/delay.h> 40#include <linux/slab.h> 41#include <linux/virtio_config.h> 42 43#include "../common/mic_dev.h" 44#include "mic_virtio.h" 45 46#define VIRTIO_SUBCODE_64 0x0D00 47 48#define MIC_MAX_VRINGS 4 49struct mic_vdev { 50 struct virtio_device vdev; 51 struct mic_device_desc __iomem *desc; 52 struct mic_device_ctrl __iomem *dc; 53 struct mic_device *mdev; 54 void __iomem *vr[MIC_MAX_VRINGS]; 55 int used_size[MIC_MAX_VRINGS]; 56 struct completion reset_done; 57 struct mic_irq *virtio_cookie; 58 int c2h_vdev_db; 59}; 60 61static struct mic_irq *virtio_config_cookie; 62#define to_micvdev(vd) container_of(vd, struct mic_vdev, vdev) 63 64/* Helper API to obtain the parent of the virtio device */ 65static inline struct device *mic_dev(struct mic_vdev *mvdev) 66{ 67 return mvdev->vdev.dev.parent; 68} 69 70/* This gets the device's feature bits. */ 71static u32 mic_get_features(struct virtio_device *vdev) 72{ 73 unsigned int i, bits; 74 u32 features = 0; 75 struct mic_device_desc __iomem *desc = to_micvdev(vdev)->desc; 76 u8 __iomem *in_features = mic_vq_features(desc); 77 int feature_len = ioread8(&desc->feature_len); 78 79 bits = min_t(unsigned, feature_len, 80 sizeof(vdev->features)) * 8; 81 for (i = 0; i < bits; i++) 82 if (ioread8(&in_features[i / 8]) & (BIT(i % 8))) 83 features |= BIT(i); 84 85 return features; 86} 87 88static void mic_finalize_features(struct virtio_device *vdev) 89{ 90 unsigned int i, bits; 91 struct mic_device_desc __iomem *desc = to_micvdev(vdev)->desc; 92 u8 feature_len = ioread8(&desc->feature_len); 93 /* Second half of bitmap is features we accept. */ 94 u8 __iomem *out_features = 95 mic_vq_features(desc) + feature_len; 96 97 /* Give virtio_ring a chance to accept features. */ 98 vring_transport_features(vdev); 99 100 memset_io(out_features, 0, feature_len); 101 bits = min_t(unsigned, feature_len, 102 sizeof(vdev->features)) * 8; 103 for (i = 0; i < bits; i++) { 104 if (test_bit(i, vdev->features)) 105 iowrite8(ioread8(&out_features[i / 8]) | (1 << (i % 8)), 106 &out_features[i / 8]); 107 } 108} 109 110/* 111 * Reading and writing elements in config space 112 */ 113static void mic_get(struct virtio_device *vdev, unsigned int offset, 114 void *buf, unsigned len) 115{ 116 struct mic_device_desc __iomem *desc = to_micvdev(vdev)->desc; 117 118 if (offset + len > ioread8(&desc->config_len)) 119 return; 120 memcpy_fromio(buf, mic_vq_configspace(desc) + offset, len); 121} 122 123static void mic_set(struct virtio_device *vdev, unsigned int offset, 124 const void *buf, unsigned len) 125{ 126 struct mic_device_desc __iomem *desc = to_micvdev(vdev)->desc; 127 128 if (offset + len > ioread8(&desc->config_len)) 129 return; 130 memcpy_toio(mic_vq_configspace(desc) + offset, buf, len); 131} 132 133/* 134 * The operations to get and set the status word just access the status 135 * field of the device descriptor. set_status also interrupts the host 136 * to tell about status changes. 137 */ 138static u8 mic_get_status(struct virtio_device *vdev) 139{ 140 return ioread8(&to_micvdev(vdev)->desc->status); 141} 142 143static void mic_set_status(struct virtio_device *vdev, u8 status) 144{ 145 struct mic_vdev *mvdev = to_micvdev(vdev); 146 if (!status) 147 return; 148 iowrite8(status, &mvdev->desc->status); 149 mic_send_intr(mvdev->mdev, mvdev->c2h_vdev_db); 150} 151 152/* Inform host on a virtio device reset and wait for ack from host */ 153static void mic_reset_inform_host(struct virtio_device *vdev) 154{ 155 struct mic_vdev *mvdev = to_micvdev(vdev); 156 struct mic_device_ctrl __iomem *dc = mvdev->dc; 157 int retry; 158 159 iowrite8(0, &dc->host_ack); 160 iowrite8(1, &dc->vdev_reset); 161 mic_send_intr(mvdev->mdev, mvdev->c2h_vdev_db); 162 163 /* Wait till host completes all card accesses and acks the reset */ 164 for (retry = 100; retry--;) { 165 if (ioread8(&dc->host_ack)) 166 break; 167 msleep(100); 168 }; 169 170 dev_dbg(mic_dev(mvdev), "%s: retry: %d\n", __func__, retry); 171 172 /* Reset status to 0 in case we timed out */ 173 iowrite8(0, &mvdev->desc->status); 174} 175 176static void mic_reset(struct virtio_device *vdev) 177{ 178 struct mic_vdev *mvdev = to_micvdev(vdev); 179 180 dev_dbg(mic_dev(mvdev), "%s: virtio id %d\n", 181 __func__, vdev->id.device); 182 183 mic_reset_inform_host(vdev); 184 complete_all(&mvdev->reset_done); 185} 186 187/* 188 * The virtio_ring code calls this API when it wants to notify the Host. 189 */ 190static bool mic_notify(struct virtqueue *vq) 191{ 192 struct mic_vdev *mvdev = vq->priv; 193 194 mic_send_intr(mvdev->mdev, mvdev->c2h_vdev_db); 195 return true; 196} 197 198static void mic_del_vq(struct virtqueue *vq, int n) 199{ 200 struct mic_vdev *mvdev = to_micvdev(vq->vdev); 201 struct vring *vr = (struct vring *)(vq + 1); 202 203 free_pages((unsigned long) vr->used, get_order(mvdev->used_size[n])); 204 vring_del_virtqueue(vq); 205 mic_card_unmap(mvdev->mdev, mvdev->vr[n]); 206 mvdev->vr[n] = NULL; 207} 208 209static void mic_del_vqs(struct virtio_device *vdev) 210{ 211 struct mic_vdev *mvdev = to_micvdev(vdev); 212 struct virtqueue *vq, *n; 213 int idx = 0; 214 215 dev_dbg(mic_dev(mvdev), "%s\n", __func__); 216 217 list_for_each_entry_safe(vq, n, &vdev->vqs, list) 218 mic_del_vq(vq, idx++); 219} 220 221/* 222 * This routine will assign vring's allocated in host/io memory. Code in 223 * virtio_ring.c however continues to access this io memory as if it were local 224 * memory without io accessors. 225 */ 226static struct virtqueue *mic_find_vq(struct virtio_device *vdev, 227 unsigned index, 228 void (*callback)(struct virtqueue *vq), 229 const char *name) 230{ 231 struct mic_vdev *mvdev = to_micvdev(vdev); 232 struct mic_vqconfig __iomem *vqconfig; 233 struct mic_vqconfig config; 234 struct virtqueue *vq; 235 void __iomem *va; 236 struct _mic_vring_info __iomem *info; 237 void *used; 238 int vr_size, _vr_size, err, magic; 239 struct vring *vr; 240 u8 type = ioread8(&mvdev->desc->type); 241 242 if (index >= ioread8(&mvdev->desc->num_vq)) 243 return ERR_PTR(-ENOENT); 244 245 if (!name) 246 return ERR_PTR(-ENOENT); 247 248 /* First assign the vring's allocated in host memory */ 249 vqconfig = mic_vq_config(mvdev->desc) + index; 250 memcpy_fromio(&config, vqconfig, sizeof(config)); 251 _vr_size = vring_size(le16_to_cpu(config.num), MIC_VIRTIO_RING_ALIGN); 252 vr_size = PAGE_ALIGN(_vr_size + sizeof(struct _mic_vring_info)); 253 va = mic_card_map(mvdev->mdev, le64_to_cpu(config.address), vr_size); 254 if (!va) 255 return ERR_PTR(-ENOMEM); 256 mvdev->vr[index] = va; 257 memset_io(va, 0x0, _vr_size); 258 vq = vring_new_virtqueue(index, le16_to_cpu(config.num), 259 MIC_VIRTIO_RING_ALIGN, vdev, false, 260 (void __force *)va, mic_notify, callback, 261 name); 262 if (!vq) { 263 err = -ENOMEM; 264 goto unmap; 265 } 266 info = va + _vr_size; 267 magic = ioread32(&info->magic); 268 269 if (WARN(magic != MIC_MAGIC + type + index, "magic mismatch")) { 270 err = -EIO; 271 goto unmap; 272 } 273 274 /* Allocate and reassign used ring now */ 275 mvdev->used_size[index] = PAGE_ALIGN(sizeof(__u16) * 3 + 276 sizeof(struct vring_used_elem) * 277 le16_to_cpu(config.num)); 278 used = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 279 get_order(mvdev->used_size[index])); 280 if (!used) { 281 err = -ENOMEM; 282 dev_err(mic_dev(mvdev), "%s %d err %d\n", 283 __func__, __LINE__, err); 284 goto del_vq; 285 } 286 iowrite64(virt_to_phys(used), &vqconfig->used_address); 287 288 /* 289 * To reassign the used ring here we are directly accessing 290 * struct vring_virtqueue which is a private data structure 291 * in virtio_ring.c. At the minimum, a BUILD_BUG_ON() in 292 * vring_new_virtqueue() would ensure that 293 * (&vq->vring == (struct vring *) (&vq->vq + 1)); 294 */ 295 vr = (struct vring *)(vq + 1); 296 vr->used = used; 297 298 vq->priv = mvdev; 299 return vq; 300del_vq: 301 vring_del_virtqueue(vq); 302unmap: 303 mic_card_unmap(mvdev->mdev, mvdev->vr[index]); 304 return ERR_PTR(err); 305} 306 307static int mic_find_vqs(struct virtio_device *vdev, unsigned nvqs, 308 struct virtqueue *vqs[], 309 vq_callback_t *callbacks[], 310 const char *names[]) 311{ 312 struct mic_vdev *mvdev = to_micvdev(vdev); 313 struct mic_device_ctrl __iomem *dc = mvdev->dc; 314 int i, err, retry; 315 316 /* We must have this many virtqueues. */ 317 if (nvqs > ioread8(&mvdev->desc->num_vq)) 318 return -ENOENT; 319 320 for (i = 0; i < nvqs; ++i) { 321 dev_dbg(mic_dev(mvdev), "%s: %d: %s\n", 322 __func__, i, names[i]); 323 vqs[i] = mic_find_vq(vdev, i, callbacks[i], names[i]); 324 if (IS_ERR(vqs[i])) { 325 err = PTR_ERR(vqs[i]); 326 goto error; 327 } 328 } 329 330 iowrite8(1, &dc->used_address_updated); 331 /* 332 * Send an interrupt to the host to inform it that used 333 * rings have been re-assigned. 334 */ 335 mic_send_intr(mvdev->mdev, mvdev->c2h_vdev_db); 336 for (retry = 100; retry--;) { 337 if (!ioread8(&dc->used_address_updated)) 338 break; 339 msleep(100); 340 }; 341 342 dev_dbg(mic_dev(mvdev), "%s: retry: %d\n", __func__, retry); 343 if (!retry) { 344 err = -ENODEV; 345 goto error; 346 } 347 348 return 0; 349error: 350 mic_del_vqs(vdev); 351 return err; 352} 353 354/* 355 * The config ops structure as defined by virtio config 356 */ 357static struct virtio_config_ops mic_vq_config_ops = { 358 .get_features = mic_get_features, 359 .finalize_features = mic_finalize_features, 360 .get = mic_get, 361 .set = mic_set, 362 .get_status = mic_get_status, 363 .set_status = mic_set_status, 364 .reset = mic_reset, 365 .find_vqs = mic_find_vqs, 366 .del_vqs = mic_del_vqs, 367}; 368 369static irqreturn_t 370mic_virtio_intr_handler(int irq, void *data) 371{ 372 struct mic_vdev *mvdev = data; 373 struct virtqueue *vq; 374 375 mic_ack_interrupt(mvdev->mdev); 376 list_for_each_entry(vq, &mvdev->vdev.vqs, list) 377 vring_interrupt(0, vq); 378 379 return IRQ_HANDLED; 380} 381 382static void mic_virtio_release_dev(struct device *_d) 383{ 384 /* 385 * No need for a release method similar to virtio PCI. 386 * Provide an empty one to avoid getting a warning from core. 387 */ 388} 389 390/* 391 * adds a new device and register it with virtio 392 * appropriate drivers are loaded by the device model 393 */ 394static int mic_add_device(struct mic_device_desc __iomem *d, 395 unsigned int offset, struct mic_driver *mdrv) 396{ 397 struct mic_vdev *mvdev; 398 int ret; 399 int virtio_db; 400 u8 type = ioread8(&d->type); 401 402 mvdev = kzalloc(sizeof(*mvdev), GFP_KERNEL); 403 if (!mvdev) { 404 dev_err(mdrv->dev, "Cannot allocate mic dev %u type %u\n", 405 offset, type); 406 return -ENOMEM; 407 } 408 409 mvdev->mdev = &mdrv->mdev; 410 mvdev->vdev.dev.parent = mdrv->dev; 411 mvdev->vdev.dev.release = mic_virtio_release_dev; 412 mvdev->vdev.id.device = type; 413 mvdev->vdev.config = &mic_vq_config_ops; 414 mvdev->desc = d; 415 mvdev->dc = (void __iomem *)d + mic_aligned_desc_size(d); 416 init_completion(&mvdev->reset_done); 417 418 virtio_db = mic_next_card_db(); 419 mvdev->virtio_cookie = mic_request_card_irq(mic_virtio_intr_handler, 420 NULL, "virtio intr", mvdev, virtio_db); 421 if (IS_ERR(mvdev->virtio_cookie)) { 422 ret = PTR_ERR(mvdev->virtio_cookie); 423 goto kfree; 424 } 425 iowrite8((u8)virtio_db, &mvdev->dc->h2c_vdev_db); 426 mvdev->c2h_vdev_db = ioread8(&mvdev->dc->c2h_vdev_db); 427 428 ret = register_virtio_device(&mvdev->vdev); 429 if (ret) { 430 dev_err(mic_dev(mvdev), 431 "Failed to register mic device %u type %u\n", 432 offset, type); 433 goto free_irq; 434 } 435 iowrite64((u64)mvdev, &mvdev->dc->vdev); 436 dev_dbg(mic_dev(mvdev), "%s: registered mic device %u type %u mvdev %p\n", 437 __func__, offset, type, mvdev); 438 439 return 0; 440 441free_irq: 442 mic_free_card_irq(mvdev->virtio_cookie, mvdev); 443kfree: 444 kfree(mvdev); 445 return ret; 446} 447 448/* 449 * match for a mic device with a specific desc pointer 450 */ 451static int mic_match_desc(struct device *dev, void *data) 452{ 453 struct virtio_device *vdev = dev_to_virtio(dev); 454 struct mic_vdev *mvdev = to_micvdev(vdev); 455 456 return mvdev->desc == (void __iomem *)data; 457} 458 459static void mic_handle_config_change(struct mic_device_desc __iomem *d, 460 unsigned int offset, struct mic_driver *mdrv) 461{ 462 struct mic_device_ctrl __iomem *dc 463 = (void __iomem *)d + mic_aligned_desc_size(d); 464 struct mic_vdev *mvdev = (struct mic_vdev *)ioread64(&dc->vdev); 465 466 if (ioread8(&dc->config_change) != MIC_VIRTIO_PARAM_CONFIG_CHANGED) 467 return; 468 469 dev_dbg(mdrv->dev, "%s %d\n", __func__, __LINE__); 470 virtio_config_changed(&mvdev->vdev); 471 iowrite8(1, &dc->guest_ack); 472} 473 474/* 475 * removes a virtio device if a hot remove event has been 476 * requested by the host. 477 */ 478static int mic_remove_device(struct mic_device_desc __iomem *d, 479 unsigned int offset, struct mic_driver *mdrv) 480{ 481 struct mic_device_ctrl __iomem *dc 482 = (void __iomem *)d + mic_aligned_desc_size(d); 483 struct mic_vdev *mvdev = (struct mic_vdev *)ioread64(&dc->vdev); 484 u8 status; 485 int ret = -1; 486 487 if (ioread8(&dc->config_change) == MIC_VIRTIO_PARAM_DEV_REMOVE) { 488 dev_dbg(mdrv->dev, 489 "%s %d config_change %d type %d mvdev %p\n", 490 __func__, __LINE__, 491 ioread8(&dc->config_change), ioread8(&d->type), mvdev); 492 493 status = ioread8(&d->status); 494 reinit_completion(&mvdev->reset_done); 495 unregister_virtio_device(&mvdev->vdev); 496 mic_free_card_irq(mvdev->virtio_cookie, mvdev); 497 if (status & VIRTIO_CONFIG_S_DRIVER_OK) 498 wait_for_completion(&mvdev->reset_done); 499 kfree(mvdev); 500 iowrite8(1, &dc->guest_ack); 501 dev_dbg(mdrv->dev, "%s %d guest_ack %d\n", 502 __func__, __LINE__, ioread8(&dc->guest_ack)); 503 ret = 0; 504 } 505 506 return ret; 507} 508 509#define REMOVE_DEVICES true 510 511static void mic_scan_devices(struct mic_driver *mdrv, bool remove) 512{ 513 s8 type; 514 unsigned int i; 515 struct mic_device_desc __iomem *d; 516 struct mic_device_ctrl __iomem *dc; 517 struct device *dev; 518 int ret; 519 520 for (i = sizeof(struct mic_bootparam); i < MIC_DP_SIZE; 521 i += mic_total_desc_size(d)) { 522 d = mdrv->dp + i; 523 dc = (void __iomem *)d + mic_aligned_desc_size(d); 524 /* 525 * This read barrier is paired with the corresponding write 526 * barrier on the host which is inserted before adding or 527 * removing a virtio device descriptor, by updating the type. 528 */ 529 rmb(); 530 type = ioread8(&d->type); 531 532 /* end of list */ 533 if (type == 0) 534 break; 535 536 if (type == -1) 537 continue; 538 539 /* device already exists */ 540 dev = device_find_child(mdrv->dev, (void __force *)d, 541 mic_match_desc); 542 if (dev) { 543 if (remove) 544 iowrite8(MIC_VIRTIO_PARAM_DEV_REMOVE, 545 &dc->config_change); 546 put_device(dev); 547 mic_handle_config_change(d, i, mdrv); 548 ret = mic_remove_device(d, i, mdrv); 549 if (!ret && !remove) 550 iowrite8(-1, &d->type); 551 if (remove) { 552 iowrite8(0, &dc->config_change); 553 iowrite8(0, &dc->guest_ack); 554 } 555 continue; 556 } 557 558 /* new device */ 559 dev_dbg(mdrv->dev, "%s %d Adding new virtio device %p\n", 560 __func__, __LINE__, d); 561 if (!remove) 562 mic_add_device(d, i, mdrv); 563 } 564} 565 566/* 567 * mic_hotplug_device tries to find changes in the device page. 568 */ 569static void mic_hotplug_devices(struct work_struct *work) 570{ 571 struct mic_driver *mdrv = container_of(work, 572 struct mic_driver, hotplug_work); 573 574 mic_scan_devices(mdrv, !REMOVE_DEVICES); 575} 576 577/* 578 * Interrupt handler for hot plug/config changes etc. 579 */ 580static irqreturn_t 581mic_extint_handler(int irq, void *data) 582{ 583 struct mic_driver *mdrv = (struct mic_driver *)data; 584 585 dev_dbg(mdrv->dev, "%s %d hotplug work\n", 586 __func__, __LINE__); 587 mic_ack_interrupt(&mdrv->mdev); 588 schedule_work(&mdrv->hotplug_work); 589 return IRQ_HANDLED; 590} 591 592/* 593 * Init function for virtio 594 */ 595int mic_devices_init(struct mic_driver *mdrv) 596{ 597 int rc; 598 struct mic_bootparam __iomem *bootparam; 599 int config_db; 600 601 INIT_WORK(&mdrv->hotplug_work, mic_hotplug_devices); 602 mic_scan_devices(mdrv, !REMOVE_DEVICES); 603 604 config_db = mic_next_card_db(); 605 virtio_config_cookie = mic_request_card_irq(mic_extint_handler, NULL, 606 "virtio_config_intr", mdrv, 607 config_db); 608 if (IS_ERR(virtio_config_cookie)) { 609 rc = PTR_ERR(virtio_config_cookie); 610 goto exit; 611 } 612 613 bootparam = mdrv->dp; 614 iowrite8(config_db, &bootparam->h2c_config_db); 615 return 0; 616exit: 617 return rc; 618} 619 620/* 621 * Uninit function for virtio 622 */ 623void mic_devices_uninit(struct mic_driver *mdrv) 624{ 625 struct mic_bootparam __iomem *bootparam = mdrv->dp; 626 iowrite8(-1, &bootparam->h2c_config_db); 627 mic_free_card_irq(virtio_config_cookie, mdrv); 628 flush_work(&mdrv->hotplug_work); 629 mic_scan_devices(mdrv, REMOVE_DEVICES); 630} 631