1/* 2 * Universal Flash Storage Host controller driver Core 3 * 4 * This code is based on drivers/scsi/ufs/ufshcd.c 5 * Copyright (C) 2011-2013 Samsung India Software Operations 6 * Copyright (c) 2013-2014, The Linux Foundation. All rights reserved. 7 * 8 * Authors: 9 * Santosh Yaraganavi <santosh.sy@samsung.com> 10 * Vinayak Holikatti <h.vinayak@samsung.com> 11 * 12 * This program is free software; you can redistribute it and/or 13 * modify it under the terms of the GNU General Public License 14 * as published by the Free Software Foundation; either version 2 15 * of the License, or (at your option) any later version. 16 * See the COPYING file in the top-level directory or visit 17 * <http://www.gnu.org/licenses/gpl-2.0.html> 18 * 19 * This program is distributed in the hope that it will be useful, 20 * but WITHOUT ANY WARRANTY; without even the implied warranty of 21 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 22 * GNU General Public License for more details. 23 * 24 * This program is provided "AS IS" and "WITH ALL FAULTS" and 25 * without warranty of any kind. You are solely responsible for 26 * determining the appropriateness of using and distributing 27 * the program and assume all risks associated with your exercise 28 * of rights with respect to the program, including but not limited 29 * to infringement of third party rights, the risks and costs of 30 * program errors, damage to or loss of data, programs or equipment, 31 * and unavailability or interruption of operations. Under no 32 * circumstances will the contributor of this Program be liable for 33 * any damages of any kind arising from your use or distribution of 34 * this program. 35 * 36 * The Linux Foundation chooses to take subject only to the GPLv2 37 * license terms, and distributes only under these terms. 38 */ 39 40#include <linux/async.h> 41#include <linux/devfreq.h> 42 43#include "ufshcd.h" 44#include "unipro.h" 45 46#define UFSHCD_ENABLE_INTRS (UTP_TRANSFER_REQ_COMPL |\ 47 UTP_TASK_REQ_COMPL |\ 48 UFSHCD_ERROR_MASK) 49/* UIC command timeout, unit: ms */ 50#define UIC_CMD_TIMEOUT 500 51 52/* NOP OUT retries waiting for NOP IN response */ 53#define NOP_OUT_RETRIES 10 54/* Timeout after 30 msecs if NOP OUT hangs without response */ 55#define NOP_OUT_TIMEOUT 30 /* msecs */ 56 57/* Query request retries */ 58#define QUERY_REQ_RETRIES 10 59/* Query request timeout */ 60#define QUERY_REQ_TIMEOUT 30 /* msec */ 61 62/* Task management command timeout */ 63#define TM_CMD_TIMEOUT 100 /* msecs */ 64 65/* maximum number of link-startup retries */ 66#define DME_LINKSTARTUP_RETRIES 3 67 68/* maximum number of reset retries before giving up */ 69#define MAX_HOST_RESET_RETRIES 5 70 71/* Expose the flag value from utp_upiu_query.value */ 72#define MASK_QUERY_UPIU_FLAG_LOC 0xFF 73 74/* Interrupt aggregation default timeout, unit: 40us */ 75#define INT_AGGR_DEF_TO 0x02 76 77#define ufshcd_toggle_vreg(_dev, _vreg, _on) \ 78 ({ \ 79 int _ret; \ 80 if (_on) \ 81 _ret = ufshcd_enable_vreg(_dev, _vreg); \ 82 else \ 83 _ret = ufshcd_disable_vreg(_dev, _vreg); \ 84 _ret; \ 85 }) 86 87static u32 ufs_query_desc_max_size[] = { 88 QUERY_DESC_DEVICE_MAX_SIZE, 89 QUERY_DESC_CONFIGURAION_MAX_SIZE, 90 QUERY_DESC_UNIT_MAX_SIZE, 91 QUERY_DESC_RFU_MAX_SIZE, 92 QUERY_DESC_INTERCONNECT_MAX_SIZE, 93 QUERY_DESC_STRING_MAX_SIZE, 94 QUERY_DESC_RFU_MAX_SIZE, 95 QUERY_DESC_GEOMETRY_MAZ_SIZE, 96 QUERY_DESC_POWER_MAX_SIZE, 97 QUERY_DESC_RFU_MAX_SIZE, 98}; 99 100enum { 101 UFSHCD_MAX_CHANNEL = 0, 102 UFSHCD_MAX_ID = 1, 103 UFSHCD_CMD_PER_LUN = 32, 104 UFSHCD_CAN_QUEUE = 32, 105}; 106 107/* UFSHCD states */ 108enum { 109 UFSHCD_STATE_RESET, 110 UFSHCD_STATE_ERROR, 111 UFSHCD_STATE_OPERATIONAL, 112}; 113 114/* UFSHCD error handling flags */ 115enum { 116 UFSHCD_EH_IN_PROGRESS = (1 << 0), 117}; 118 119/* UFSHCD UIC layer error flags */ 120enum { 121 UFSHCD_UIC_DL_PA_INIT_ERROR = (1 << 0), /* Data link layer error */ 122 UFSHCD_UIC_NL_ERROR = (1 << 1), /* Network layer error */ 123 UFSHCD_UIC_TL_ERROR = (1 << 2), /* Transport Layer error */ 124 UFSHCD_UIC_DME_ERROR = (1 << 3), /* DME error */ 125}; 126 127/* Interrupt configuration options */ 128enum { 129 UFSHCD_INT_DISABLE, 130 UFSHCD_INT_ENABLE, 131 UFSHCD_INT_CLEAR, 132}; 133 134#define ufshcd_set_eh_in_progress(h) \ 135 (h->eh_flags |= UFSHCD_EH_IN_PROGRESS) 136#define ufshcd_eh_in_progress(h) \ 137 (h->eh_flags & UFSHCD_EH_IN_PROGRESS) 138#define ufshcd_clear_eh_in_progress(h) \ 139 (h->eh_flags &= ~UFSHCD_EH_IN_PROGRESS) 140 141#define ufshcd_set_ufs_dev_active(h) \ 142 ((h)->curr_dev_pwr_mode = UFS_ACTIVE_PWR_MODE) 143#define ufshcd_set_ufs_dev_sleep(h) \ 144 ((h)->curr_dev_pwr_mode = UFS_SLEEP_PWR_MODE) 145#define ufshcd_set_ufs_dev_poweroff(h) \ 146 ((h)->curr_dev_pwr_mode = UFS_POWERDOWN_PWR_MODE) 147#define ufshcd_is_ufs_dev_active(h) \ 148 ((h)->curr_dev_pwr_mode == UFS_ACTIVE_PWR_MODE) 149#define ufshcd_is_ufs_dev_sleep(h) \ 150 ((h)->curr_dev_pwr_mode == UFS_SLEEP_PWR_MODE) 151#define ufshcd_is_ufs_dev_poweroff(h) \ 152 ((h)->curr_dev_pwr_mode == UFS_POWERDOWN_PWR_MODE) 153 154static struct ufs_pm_lvl_states ufs_pm_lvl_states[] = { 155 {UFS_ACTIVE_PWR_MODE, UIC_LINK_ACTIVE_STATE}, 156 {UFS_ACTIVE_PWR_MODE, UIC_LINK_HIBERN8_STATE}, 157 {UFS_SLEEP_PWR_MODE, UIC_LINK_ACTIVE_STATE}, 158 {UFS_SLEEP_PWR_MODE, UIC_LINK_HIBERN8_STATE}, 159 {UFS_POWERDOWN_PWR_MODE, UIC_LINK_HIBERN8_STATE}, 160 {UFS_POWERDOWN_PWR_MODE, UIC_LINK_OFF_STATE}, 161}; 162 163static inline enum ufs_dev_pwr_mode 164ufs_get_pm_lvl_to_dev_pwr_mode(enum ufs_pm_level lvl) 165{ 166 return ufs_pm_lvl_states[lvl].dev_state; 167} 168 169static inline enum uic_link_state 170ufs_get_pm_lvl_to_link_pwr_state(enum ufs_pm_level lvl) 171{ 172 return ufs_pm_lvl_states[lvl].link_state; 173} 174 175static void ufshcd_tmc_handler(struct ufs_hba *hba); 176static void ufshcd_async_scan(void *data, async_cookie_t cookie); 177static int ufshcd_reset_and_restore(struct ufs_hba *hba); 178static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag); 179static void ufshcd_hba_exit(struct ufs_hba *hba); 180static int ufshcd_probe_hba(struct ufs_hba *hba); 181static int __ufshcd_setup_clocks(struct ufs_hba *hba, bool on, 182 bool skip_ref_clk); 183static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on); 184static int ufshcd_uic_hibern8_exit(struct ufs_hba *hba); 185static int ufshcd_uic_hibern8_enter(struct ufs_hba *hba); 186static int ufshcd_host_reset_and_restore(struct ufs_hba *hba); 187static irqreturn_t ufshcd_intr(int irq, void *__hba); 188static int ufshcd_config_pwr_mode(struct ufs_hba *hba, 189 struct ufs_pa_layer_attr *desired_pwr_mode); 190 191static inline int ufshcd_enable_irq(struct ufs_hba *hba) 192{ 193 int ret = 0; 194 195 if (!hba->is_irq_enabled) { 196 ret = request_irq(hba->irq, ufshcd_intr, IRQF_SHARED, UFSHCD, 197 hba); 198 if (ret) 199 dev_err(hba->dev, "%s: request_irq failed, ret=%d\n", 200 __func__, ret); 201 hba->is_irq_enabled = true; 202 } 203 204 return ret; 205} 206 207static inline void ufshcd_disable_irq(struct ufs_hba *hba) 208{ 209 if (hba->is_irq_enabled) { 210 free_irq(hba->irq, hba); 211 hba->is_irq_enabled = false; 212 } 213} 214 215/* 216 * ufshcd_wait_for_register - wait for register value to change 217 * @hba - per-adapter interface 218 * @reg - mmio register offset 219 * @mask - mask to apply to read register value 220 * @val - wait condition 221 * @interval_us - polling interval in microsecs 222 * @timeout_ms - timeout in millisecs 223 * 224 * Returns -ETIMEDOUT on error, zero on success 225 */ 226static int ufshcd_wait_for_register(struct ufs_hba *hba, u32 reg, u32 mask, 227 u32 val, unsigned long interval_us, unsigned long timeout_ms) 228{ 229 int err = 0; 230 unsigned long timeout = jiffies + msecs_to_jiffies(timeout_ms); 231 232 /* ignore bits that we don't intend to wait on */ 233 val = val & mask; 234 235 while ((ufshcd_readl(hba, reg) & mask) != val) { 236 /* wakeup within 50us of expiry */ 237 usleep_range(interval_us, interval_us + 50); 238 239 if (time_after(jiffies, timeout)) { 240 if ((ufshcd_readl(hba, reg) & mask) != val) 241 err = -ETIMEDOUT; 242 break; 243 } 244 } 245 246 return err; 247} 248 249/** 250 * ufshcd_get_intr_mask - Get the interrupt bit mask 251 * @hba - Pointer to adapter instance 252 * 253 * Returns interrupt bit mask per version 254 */ 255static inline u32 ufshcd_get_intr_mask(struct ufs_hba *hba) 256{ 257 if (hba->ufs_version == UFSHCI_VERSION_10) 258 return INTERRUPT_MASK_ALL_VER_10; 259 else 260 return INTERRUPT_MASK_ALL_VER_11; 261} 262 263/** 264 * ufshcd_get_ufs_version - Get the UFS version supported by the HBA 265 * @hba - Pointer to adapter instance 266 * 267 * Returns UFSHCI version supported by the controller 268 */ 269static inline u32 ufshcd_get_ufs_version(struct ufs_hba *hba) 270{ 271 return ufshcd_readl(hba, REG_UFS_VERSION); 272} 273 274/** 275 * ufshcd_is_device_present - Check if any device connected to 276 * the host controller 277 * @hba: pointer to adapter instance 278 * 279 * Returns 1 if device present, 0 if no device detected 280 */ 281static inline int ufshcd_is_device_present(struct ufs_hba *hba) 282{ 283 return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) & 284 DEVICE_PRESENT) ? 1 : 0; 285} 286 287/** 288 * ufshcd_get_tr_ocs - Get the UTRD Overall Command Status 289 * @lrb: pointer to local command reference block 290 * 291 * This function is used to get the OCS field from UTRD 292 * Returns the OCS field in the UTRD 293 */ 294static inline int ufshcd_get_tr_ocs(struct ufshcd_lrb *lrbp) 295{ 296 return le32_to_cpu(lrbp->utr_descriptor_ptr->header.dword_2) & MASK_OCS; 297} 298 299/** 300 * ufshcd_get_tmr_ocs - Get the UTMRD Overall Command Status 301 * @task_req_descp: pointer to utp_task_req_desc structure 302 * 303 * This function is used to get the OCS field from UTMRD 304 * Returns the OCS field in the UTMRD 305 */ 306static inline int 307ufshcd_get_tmr_ocs(struct utp_task_req_desc *task_req_descp) 308{ 309 return le32_to_cpu(task_req_descp->header.dword_2) & MASK_OCS; 310} 311 312/** 313 * ufshcd_get_tm_free_slot - get a free slot for task management request 314 * @hba: per adapter instance 315 * @free_slot: pointer to variable with available slot value 316 * 317 * Get a free tag and lock it until ufshcd_put_tm_slot() is called. 318 * Returns 0 if free slot is not available, else return 1 with tag value 319 * in @free_slot. 320 */ 321static bool ufshcd_get_tm_free_slot(struct ufs_hba *hba, int *free_slot) 322{ 323 int tag; 324 bool ret = false; 325 326 if (!free_slot) 327 goto out; 328 329 do { 330 tag = find_first_zero_bit(&hba->tm_slots_in_use, hba->nutmrs); 331 if (tag >= hba->nutmrs) 332 goto out; 333 } while (test_and_set_bit_lock(tag, &hba->tm_slots_in_use)); 334 335 *free_slot = tag; 336 ret = true; 337out: 338 return ret; 339} 340 341static inline void ufshcd_put_tm_slot(struct ufs_hba *hba, int slot) 342{ 343 clear_bit_unlock(slot, &hba->tm_slots_in_use); 344} 345 346/** 347 * ufshcd_utrl_clear - Clear a bit in UTRLCLR register 348 * @hba: per adapter instance 349 * @pos: position of the bit to be cleared 350 */ 351static inline void ufshcd_utrl_clear(struct ufs_hba *hba, u32 pos) 352{ 353 ufshcd_writel(hba, ~(1 << pos), REG_UTP_TRANSFER_REQ_LIST_CLEAR); 354} 355 356/** 357 * ufshcd_get_lists_status - Check UCRDY, UTRLRDY and UTMRLRDY 358 * @reg: Register value of host controller status 359 * 360 * Returns integer, 0 on Success and positive value if failed 361 */ 362static inline int ufshcd_get_lists_status(u32 reg) 363{ 364 /* 365 * The mask 0xFF is for the following HCS register bits 366 * Bit Description 367 * 0 Device Present 368 * 1 UTRLRDY 369 * 2 UTMRLRDY 370 * 3 UCRDY 371 * 4 HEI 372 * 5 DEI 373 * 6-7 reserved 374 */ 375 return (((reg) & (0xFF)) >> 1) ^ (0x07); 376} 377 378/** 379 * ufshcd_get_uic_cmd_result - Get the UIC command result 380 * @hba: Pointer to adapter instance 381 * 382 * This function gets the result of UIC command completion 383 * Returns 0 on success, non zero value on error 384 */ 385static inline int ufshcd_get_uic_cmd_result(struct ufs_hba *hba) 386{ 387 return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2) & 388 MASK_UIC_COMMAND_RESULT; 389} 390 391/** 392 * ufshcd_get_dme_attr_val - Get the value of attribute returned by UIC command 393 * @hba: Pointer to adapter instance 394 * 395 * This function gets UIC command argument3 396 * Returns 0 on success, non zero value on error 397 */ 398static inline u32 ufshcd_get_dme_attr_val(struct ufs_hba *hba) 399{ 400 return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3); 401} 402 403/** 404 * ufshcd_get_req_rsp - returns the TR response transaction type 405 * @ucd_rsp_ptr: pointer to response UPIU 406 */ 407static inline int 408ufshcd_get_req_rsp(struct utp_upiu_rsp *ucd_rsp_ptr) 409{ 410 return be32_to_cpu(ucd_rsp_ptr->header.dword_0) >> 24; 411} 412 413/** 414 * ufshcd_get_rsp_upiu_result - Get the result from response UPIU 415 * @ucd_rsp_ptr: pointer to response UPIU 416 * 417 * This function gets the response status and scsi_status from response UPIU 418 * Returns the response result code. 419 */ 420static inline int 421ufshcd_get_rsp_upiu_result(struct utp_upiu_rsp *ucd_rsp_ptr) 422{ 423 return be32_to_cpu(ucd_rsp_ptr->header.dword_1) & MASK_RSP_UPIU_RESULT; 424} 425 426/* 427 * ufshcd_get_rsp_upiu_data_seg_len - Get the data segment length 428 * from response UPIU 429 * @ucd_rsp_ptr: pointer to response UPIU 430 * 431 * Return the data segment length. 432 */ 433static inline unsigned int 434ufshcd_get_rsp_upiu_data_seg_len(struct utp_upiu_rsp *ucd_rsp_ptr) 435{ 436 return be32_to_cpu(ucd_rsp_ptr->header.dword_2) & 437 MASK_RSP_UPIU_DATA_SEG_LEN; 438} 439 440/** 441 * ufshcd_is_exception_event - Check if the device raised an exception event 442 * @ucd_rsp_ptr: pointer to response UPIU 443 * 444 * The function checks if the device raised an exception event indicated in 445 * the Device Information field of response UPIU. 446 * 447 * Returns true if exception is raised, false otherwise. 448 */ 449static inline bool ufshcd_is_exception_event(struct utp_upiu_rsp *ucd_rsp_ptr) 450{ 451 return be32_to_cpu(ucd_rsp_ptr->header.dword_2) & 452 MASK_RSP_EXCEPTION_EVENT ? true : false; 453} 454 455/** 456 * ufshcd_reset_intr_aggr - Reset interrupt aggregation values. 457 * @hba: per adapter instance 458 */ 459static inline void 460ufshcd_reset_intr_aggr(struct ufs_hba *hba) 461{ 462 ufshcd_writel(hba, INT_AGGR_ENABLE | 463 INT_AGGR_COUNTER_AND_TIMER_RESET, 464 REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL); 465} 466 467/** 468 * ufshcd_config_intr_aggr - Configure interrupt aggregation values. 469 * @hba: per adapter instance 470 * @cnt: Interrupt aggregation counter threshold 471 * @tmout: Interrupt aggregation timeout value 472 */ 473static inline void 474ufshcd_config_intr_aggr(struct ufs_hba *hba, u8 cnt, u8 tmout) 475{ 476 ufshcd_writel(hba, INT_AGGR_ENABLE | INT_AGGR_PARAM_WRITE | 477 INT_AGGR_COUNTER_THLD_VAL(cnt) | 478 INT_AGGR_TIMEOUT_VAL(tmout), 479 REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL); 480} 481 482/** 483 * ufshcd_enable_run_stop_reg - Enable run-stop registers, 484 * When run-stop registers are set to 1, it indicates the 485 * host controller that it can process the requests 486 * @hba: per adapter instance 487 */ 488static void ufshcd_enable_run_stop_reg(struct ufs_hba *hba) 489{ 490 ufshcd_writel(hba, UTP_TASK_REQ_LIST_RUN_STOP_BIT, 491 REG_UTP_TASK_REQ_LIST_RUN_STOP); 492 ufshcd_writel(hba, UTP_TRANSFER_REQ_LIST_RUN_STOP_BIT, 493 REG_UTP_TRANSFER_REQ_LIST_RUN_STOP); 494} 495 496/** 497 * ufshcd_hba_start - Start controller initialization sequence 498 * @hba: per adapter instance 499 */ 500static inline void ufshcd_hba_start(struct ufs_hba *hba) 501{ 502 ufshcd_writel(hba, CONTROLLER_ENABLE, REG_CONTROLLER_ENABLE); 503} 504 505/** 506 * ufshcd_is_hba_active - Get controller state 507 * @hba: per adapter instance 508 * 509 * Returns zero if controller is active, 1 otherwise 510 */ 511static inline int ufshcd_is_hba_active(struct ufs_hba *hba) 512{ 513 return (ufshcd_readl(hba, REG_CONTROLLER_ENABLE) & 0x1) ? 0 : 1; 514} 515 516static void ufshcd_ungate_work(struct work_struct *work) 517{ 518 int ret; 519 unsigned long flags; 520 struct ufs_hba *hba = container_of(work, struct ufs_hba, 521 clk_gating.ungate_work); 522 523 cancel_delayed_work_sync(&hba->clk_gating.gate_work); 524 525 spin_lock_irqsave(hba->host->host_lock, flags); 526 if (hba->clk_gating.state == CLKS_ON) { 527 spin_unlock_irqrestore(hba->host->host_lock, flags); 528 goto unblock_reqs; 529 } 530 531 spin_unlock_irqrestore(hba->host->host_lock, flags); 532 ufshcd_setup_clocks(hba, true); 533 534 /* Exit from hibern8 */ 535 if (ufshcd_can_hibern8_during_gating(hba)) { 536 /* Prevent gating in this path */ 537 hba->clk_gating.is_suspended = true; 538 if (ufshcd_is_link_hibern8(hba)) { 539 ret = ufshcd_uic_hibern8_exit(hba); 540 if (ret) 541 dev_err(hba->dev, "%s: hibern8 exit failed %d\n", 542 __func__, ret); 543 else 544 ufshcd_set_link_active(hba); 545 } 546 hba->clk_gating.is_suspended = false; 547 } 548unblock_reqs: 549 if (ufshcd_is_clkscaling_enabled(hba)) 550 devfreq_resume_device(hba->devfreq); 551 scsi_unblock_requests(hba->host); 552} 553 554/** 555 * ufshcd_hold - Enable clocks that were gated earlier due to ufshcd_release. 556 * Also, exit from hibern8 mode and set the link as active. 557 * @hba: per adapter instance 558 * @async: This indicates whether caller should ungate clocks asynchronously. 559 */ 560int ufshcd_hold(struct ufs_hba *hba, bool async) 561{ 562 int rc = 0; 563 unsigned long flags; 564 565 if (!ufshcd_is_clkgating_allowed(hba)) 566 goto out; 567 spin_lock_irqsave(hba->host->host_lock, flags); 568 hba->clk_gating.active_reqs++; 569 570start: 571 switch (hba->clk_gating.state) { 572 case CLKS_ON: 573 break; 574 case REQ_CLKS_OFF: 575 if (cancel_delayed_work(&hba->clk_gating.gate_work)) { 576 hba->clk_gating.state = CLKS_ON; 577 break; 578 } 579 /* 580 * If we here, it means gating work is either done or 581 * currently running. Hence, fall through to cancel gating 582 * work and to enable clocks. 583 */ 584 case CLKS_OFF: 585 scsi_block_requests(hba->host); 586 hba->clk_gating.state = REQ_CLKS_ON; 587 schedule_work(&hba->clk_gating.ungate_work); 588 /* 589 * fall through to check if we should wait for this 590 * work to be done or not. 591 */ 592 case REQ_CLKS_ON: 593 if (async) { 594 rc = -EAGAIN; 595 hba->clk_gating.active_reqs--; 596 break; 597 } 598 599 spin_unlock_irqrestore(hba->host->host_lock, flags); 600 flush_work(&hba->clk_gating.ungate_work); 601 /* Make sure state is CLKS_ON before returning */ 602 spin_lock_irqsave(hba->host->host_lock, flags); 603 goto start; 604 default: 605 dev_err(hba->dev, "%s: clk gating is in invalid state %d\n", 606 __func__, hba->clk_gating.state); 607 break; 608 } 609 spin_unlock_irqrestore(hba->host->host_lock, flags); 610out: 611 return rc; 612} 613 614static void ufshcd_gate_work(struct work_struct *work) 615{ 616 struct ufs_hba *hba = container_of(work, struct ufs_hba, 617 clk_gating.gate_work.work); 618 unsigned long flags; 619 620 spin_lock_irqsave(hba->host->host_lock, flags); 621 if (hba->clk_gating.is_suspended) { 622 hba->clk_gating.state = CLKS_ON; 623 goto rel_lock; 624 } 625 626 if (hba->clk_gating.active_reqs 627 || hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL 628 || hba->lrb_in_use || hba->outstanding_tasks 629 || hba->active_uic_cmd || hba->uic_async_done) 630 goto rel_lock; 631 632 spin_unlock_irqrestore(hba->host->host_lock, flags); 633 634 /* put the link into hibern8 mode before turning off clocks */ 635 if (ufshcd_can_hibern8_during_gating(hba)) { 636 if (ufshcd_uic_hibern8_enter(hba)) { 637 hba->clk_gating.state = CLKS_ON; 638 goto out; 639 } 640 ufshcd_set_link_hibern8(hba); 641 } 642 643 if (ufshcd_is_clkscaling_enabled(hba)) { 644 devfreq_suspend_device(hba->devfreq); 645 hba->clk_scaling.window_start_t = 0; 646 } 647 648 if (!ufshcd_is_link_active(hba)) 649 ufshcd_setup_clocks(hba, false); 650 else 651 /* If link is active, device ref_clk can't be switched off */ 652 __ufshcd_setup_clocks(hba, false, true); 653 654 /* 655 * In case you are here to cancel this work the gating state 656 * would be marked as REQ_CLKS_ON. In this case keep the state 657 * as REQ_CLKS_ON which would anyway imply that clocks are off 658 * and a request to turn them on is pending. By doing this way, 659 * we keep the state machine in tact and this would ultimately 660 * prevent from doing cancel work multiple times when there are 661 * new requests arriving before the current cancel work is done. 662 */ 663 spin_lock_irqsave(hba->host->host_lock, flags); 664 if (hba->clk_gating.state == REQ_CLKS_OFF) 665 hba->clk_gating.state = CLKS_OFF; 666 667rel_lock: 668 spin_unlock_irqrestore(hba->host->host_lock, flags); 669out: 670 return; 671} 672 673/* host lock must be held before calling this variant */ 674static void __ufshcd_release(struct ufs_hba *hba) 675{ 676 if (!ufshcd_is_clkgating_allowed(hba)) 677 return; 678 679 hba->clk_gating.active_reqs--; 680 681 if (hba->clk_gating.active_reqs || hba->clk_gating.is_suspended 682 || hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL 683 || hba->lrb_in_use || hba->outstanding_tasks 684 || hba->active_uic_cmd || hba->uic_async_done) 685 return; 686 687 hba->clk_gating.state = REQ_CLKS_OFF; 688 schedule_delayed_work(&hba->clk_gating.gate_work, 689 msecs_to_jiffies(hba->clk_gating.delay_ms)); 690} 691 692void ufshcd_release(struct ufs_hba *hba) 693{ 694 unsigned long flags; 695 696 spin_lock_irqsave(hba->host->host_lock, flags); 697 __ufshcd_release(hba); 698 spin_unlock_irqrestore(hba->host->host_lock, flags); 699} 700 701static ssize_t ufshcd_clkgate_delay_show(struct device *dev, 702 struct device_attribute *attr, char *buf) 703{ 704 struct ufs_hba *hba = dev_get_drvdata(dev); 705 706 return snprintf(buf, PAGE_SIZE, "%lu\n", hba->clk_gating.delay_ms); 707} 708 709static ssize_t ufshcd_clkgate_delay_store(struct device *dev, 710 struct device_attribute *attr, const char *buf, size_t count) 711{ 712 struct ufs_hba *hba = dev_get_drvdata(dev); 713 unsigned long flags, value; 714 715 if (kstrtoul(buf, 0, &value)) 716 return -EINVAL; 717 718 spin_lock_irqsave(hba->host->host_lock, flags); 719 hba->clk_gating.delay_ms = value; 720 spin_unlock_irqrestore(hba->host->host_lock, flags); 721 return count; 722} 723 724static void ufshcd_init_clk_gating(struct ufs_hba *hba) 725{ 726 if (!ufshcd_is_clkgating_allowed(hba)) 727 return; 728 729 hba->clk_gating.delay_ms = 150; 730 INIT_DELAYED_WORK(&hba->clk_gating.gate_work, ufshcd_gate_work); 731 INIT_WORK(&hba->clk_gating.ungate_work, ufshcd_ungate_work); 732 733 hba->clk_gating.delay_attr.show = ufshcd_clkgate_delay_show; 734 hba->clk_gating.delay_attr.store = ufshcd_clkgate_delay_store; 735 sysfs_attr_init(&hba->clk_gating.delay_attr.attr); 736 hba->clk_gating.delay_attr.attr.name = "clkgate_delay_ms"; 737 hba->clk_gating.delay_attr.attr.mode = S_IRUGO | S_IWUSR; 738 if (device_create_file(hba->dev, &hba->clk_gating.delay_attr)) 739 dev_err(hba->dev, "Failed to create sysfs for clkgate_delay\n"); 740} 741 742static void ufshcd_exit_clk_gating(struct ufs_hba *hba) 743{ 744 if (!ufshcd_is_clkgating_allowed(hba)) 745 return; 746 device_remove_file(hba->dev, &hba->clk_gating.delay_attr); 747 cancel_work_sync(&hba->clk_gating.ungate_work); 748 cancel_delayed_work_sync(&hba->clk_gating.gate_work); 749} 750 751/* Must be called with host lock acquired */ 752static void ufshcd_clk_scaling_start_busy(struct ufs_hba *hba) 753{ 754 if (!ufshcd_is_clkscaling_enabled(hba)) 755 return; 756 757 if (!hba->clk_scaling.is_busy_started) { 758 hba->clk_scaling.busy_start_t = ktime_get(); 759 hba->clk_scaling.is_busy_started = true; 760 } 761} 762 763static void ufshcd_clk_scaling_update_busy(struct ufs_hba *hba) 764{ 765 struct ufs_clk_scaling *scaling = &hba->clk_scaling; 766 767 if (!ufshcd_is_clkscaling_enabled(hba)) 768 return; 769 770 if (!hba->outstanding_reqs && scaling->is_busy_started) { 771 scaling->tot_busy_t += ktime_to_us(ktime_sub(ktime_get(), 772 scaling->busy_start_t)); 773 scaling->busy_start_t = ktime_set(0, 0); 774 scaling->is_busy_started = false; 775 } 776} 777/** 778 * ufshcd_send_command - Send SCSI or device management commands 779 * @hba: per adapter instance 780 * @task_tag: Task tag of the command 781 */ 782static inline 783void ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag) 784{ 785 ufshcd_clk_scaling_start_busy(hba); 786 __set_bit(task_tag, &hba->outstanding_reqs); 787 ufshcd_writel(hba, 1 << task_tag, REG_UTP_TRANSFER_REQ_DOOR_BELL); 788} 789 790/** 791 * ufshcd_copy_sense_data - Copy sense data in case of check condition 792 * @lrb - pointer to local reference block 793 */ 794static inline void ufshcd_copy_sense_data(struct ufshcd_lrb *lrbp) 795{ 796 int len; 797 if (lrbp->sense_buffer && 798 ufshcd_get_rsp_upiu_data_seg_len(lrbp->ucd_rsp_ptr)) { 799 len = be16_to_cpu(lrbp->ucd_rsp_ptr->sr.sense_data_len); 800 memcpy(lrbp->sense_buffer, 801 lrbp->ucd_rsp_ptr->sr.sense_data, 802 min_t(int, len, SCSI_SENSE_BUFFERSIZE)); 803 } 804} 805 806/** 807 * ufshcd_copy_query_response() - Copy the Query Response and the data 808 * descriptor 809 * @hba: per adapter instance 810 * @lrb - pointer to local reference block 811 */ 812static 813int ufshcd_copy_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) 814{ 815 struct ufs_query_res *query_res = &hba->dev_cmd.query.response; 816 817 memcpy(&query_res->upiu_res, &lrbp->ucd_rsp_ptr->qr, QUERY_OSF_SIZE); 818 819 /* Get the descriptor */ 820 if (lrbp->ucd_rsp_ptr->qr.opcode == UPIU_QUERY_OPCODE_READ_DESC) { 821 u8 *descp = (u8 *)lrbp->ucd_rsp_ptr + 822 GENERAL_UPIU_REQUEST_SIZE; 823 u16 resp_len; 824 u16 buf_len; 825 826 /* data segment length */ 827 resp_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) & 828 MASK_QUERY_DATA_SEG_LEN; 829 buf_len = be16_to_cpu( 830 hba->dev_cmd.query.request.upiu_req.length); 831 if (likely(buf_len >= resp_len)) { 832 memcpy(hba->dev_cmd.query.descriptor, descp, resp_len); 833 } else { 834 dev_warn(hba->dev, 835 "%s: Response size is bigger than buffer", 836 __func__); 837 return -EINVAL; 838 } 839 } 840 841 return 0; 842} 843 844/** 845 * ufshcd_hba_capabilities - Read controller capabilities 846 * @hba: per adapter instance 847 */ 848static inline void ufshcd_hba_capabilities(struct ufs_hba *hba) 849{ 850 hba->capabilities = ufshcd_readl(hba, REG_CONTROLLER_CAPABILITIES); 851 852 /* nutrs and nutmrs are 0 based values */ 853 hba->nutrs = (hba->capabilities & MASK_TRANSFER_REQUESTS_SLOTS) + 1; 854 hba->nutmrs = 855 ((hba->capabilities & MASK_TASK_MANAGEMENT_REQUEST_SLOTS) >> 16) + 1; 856} 857 858/** 859 * ufshcd_ready_for_uic_cmd - Check if controller is ready 860 * to accept UIC commands 861 * @hba: per adapter instance 862 * Return true on success, else false 863 */ 864static inline bool ufshcd_ready_for_uic_cmd(struct ufs_hba *hba) 865{ 866 if (ufshcd_readl(hba, REG_CONTROLLER_STATUS) & UIC_COMMAND_READY) 867 return true; 868 else 869 return false; 870} 871 872/** 873 * ufshcd_get_upmcrs - Get the power mode change request status 874 * @hba: Pointer to adapter instance 875 * 876 * This function gets the UPMCRS field of HCS register 877 * Returns value of UPMCRS field 878 */ 879static inline u8 ufshcd_get_upmcrs(struct ufs_hba *hba) 880{ 881 return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) >> 8) & 0x7; 882} 883 884/** 885 * ufshcd_dispatch_uic_cmd - Dispatch UIC commands to unipro layers 886 * @hba: per adapter instance 887 * @uic_cmd: UIC command 888 * 889 * Mutex must be held. 890 */ 891static inline void 892ufshcd_dispatch_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd) 893{ 894 WARN_ON(hba->active_uic_cmd); 895 896 hba->active_uic_cmd = uic_cmd; 897 898 /* Write Args */ 899 ufshcd_writel(hba, uic_cmd->argument1, REG_UIC_COMMAND_ARG_1); 900 ufshcd_writel(hba, uic_cmd->argument2, REG_UIC_COMMAND_ARG_2); 901 ufshcd_writel(hba, uic_cmd->argument3, REG_UIC_COMMAND_ARG_3); 902 903 /* Write UIC Cmd */ 904 ufshcd_writel(hba, uic_cmd->command & COMMAND_OPCODE_MASK, 905 REG_UIC_COMMAND); 906} 907 908/** 909 * ufshcd_wait_for_uic_cmd - Wait complectioin of UIC command 910 * @hba: per adapter instance 911 * @uic_command: UIC command 912 * 913 * Must be called with mutex held. 914 * Returns 0 only if success. 915 */ 916static int 917ufshcd_wait_for_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd) 918{ 919 int ret; 920 unsigned long flags; 921 922 if (wait_for_completion_timeout(&uic_cmd->done, 923 msecs_to_jiffies(UIC_CMD_TIMEOUT))) 924 ret = uic_cmd->argument2 & MASK_UIC_COMMAND_RESULT; 925 else 926 ret = -ETIMEDOUT; 927 928 spin_lock_irqsave(hba->host->host_lock, flags); 929 hba->active_uic_cmd = NULL; 930 spin_unlock_irqrestore(hba->host->host_lock, flags); 931 932 return ret; 933} 934 935/** 936 * __ufshcd_send_uic_cmd - Send UIC commands and retrieve the result 937 * @hba: per adapter instance 938 * @uic_cmd: UIC command 939 * 940 * Identical to ufshcd_send_uic_cmd() expect mutex. Must be called 941 * with mutex held and host_lock locked. 942 * Returns 0 only if success. 943 */ 944static int 945__ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd) 946{ 947 if (!ufshcd_ready_for_uic_cmd(hba)) { 948 dev_err(hba->dev, 949 "Controller not ready to accept UIC commands\n"); 950 return -EIO; 951 } 952 953 init_completion(&uic_cmd->done); 954 955 ufshcd_dispatch_uic_cmd(hba, uic_cmd); 956 957 return 0; 958} 959 960/** 961 * ufshcd_send_uic_cmd - Send UIC commands and retrieve the result 962 * @hba: per adapter instance 963 * @uic_cmd: UIC command 964 * 965 * Returns 0 only if success. 966 */ 967static int 968ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd) 969{ 970 int ret; 971 unsigned long flags; 972 973 ufshcd_hold(hba, false); 974 mutex_lock(&hba->uic_cmd_mutex); 975 spin_lock_irqsave(hba->host->host_lock, flags); 976 ret = __ufshcd_send_uic_cmd(hba, uic_cmd); 977 spin_unlock_irqrestore(hba->host->host_lock, flags); 978 if (!ret) 979 ret = ufshcd_wait_for_uic_cmd(hba, uic_cmd); 980 981 mutex_unlock(&hba->uic_cmd_mutex); 982 983 ufshcd_release(hba); 984 return ret; 985} 986 987/** 988 * ufshcd_map_sg - Map scatter-gather list to prdt 989 * @lrbp - pointer to local reference block 990 * 991 * Returns 0 in case of success, non-zero value in case of failure 992 */ 993static int ufshcd_map_sg(struct ufshcd_lrb *lrbp) 994{ 995 struct ufshcd_sg_entry *prd_table; 996 struct scatterlist *sg; 997 struct scsi_cmnd *cmd; 998 int sg_segments; 999 int i; 1000 1001 cmd = lrbp->cmd; 1002 sg_segments = scsi_dma_map(cmd); 1003 if (sg_segments < 0) 1004 return sg_segments; 1005 1006 if (sg_segments) { 1007 lrbp->utr_descriptor_ptr->prd_table_length = 1008 cpu_to_le16((u16) (sg_segments)); 1009 1010 prd_table = (struct ufshcd_sg_entry *)lrbp->ucd_prdt_ptr; 1011 1012 scsi_for_each_sg(cmd, sg, sg_segments, i) { 1013 prd_table[i].size = 1014 cpu_to_le32(((u32) sg_dma_len(sg))-1); 1015 prd_table[i].base_addr = 1016 cpu_to_le32(lower_32_bits(sg->dma_address)); 1017 prd_table[i].upper_addr = 1018 cpu_to_le32(upper_32_bits(sg->dma_address)); 1019 } 1020 } else { 1021 lrbp->utr_descriptor_ptr->prd_table_length = 0; 1022 } 1023 1024 return 0; 1025} 1026 1027/** 1028 * ufshcd_enable_intr - enable interrupts 1029 * @hba: per adapter instance 1030 * @intrs: interrupt bits 1031 */ 1032static void ufshcd_enable_intr(struct ufs_hba *hba, u32 intrs) 1033{ 1034 u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE); 1035 1036 if (hba->ufs_version == UFSHCI_VERSION_10) { 1037 u32 rw; 1038 rw = set & INTERRUPT_MASK_RW_VER_10; 1039 set = rw | ((set ^ intrs) & intrs); 1040 } else { 1041 set |= intrs; 1042 } 1043 1044 ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE); 1045} 1046 1047/** 1048 * ufshcd_disable_intr - disable interrupts 1049 * @hba: per adapter instance 1050 * @intrs: interrupt bits 1051 */ 1052static void ufshcd_disable_intr(struct ufs_hba *hba, u32 intrs) 1053{ 1054 u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE); 1055 1056 if (hba->ufs_version == UFSHCI_VERSION_10) { 1057 u32 rw; 1058 rw = (set & INTERRUPT_MASK_RW_VER_10) & 1059 ~(intrs & INTERRUPT_MASK_RW_VER_10); 1060 set = rw | ((set & intrs) & ~INTERRUPT_MASK_RW_VER_10); 1061 1062 } else { 1063 set &= ~intrs; 1064 } 1065 1066 ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE); 1067} 1068 1069/** 1070 * ufshcd_prepare_req_desc_hdr() - Fills the requests header 1071 * descriptor according to request 1072 * @lrbp: pointer to local reference block 1073 * @upiu_flags: flags required in the header 1074 * @cmd_dir: requests data direction 1075 */ 1076static void ufshcd_prepare_req_desc_hdr(struct ufshcd_lrb *lrbp, 1077 u32 *upiu_flags, enum dma_data_direction cmd_dir) 1078{ 1079 struct utp_transfer_req_desc *req_desc = lrbp->utr_descriptor_ptr; 1080 u32 data_direction; 1081 u32 dword_0; 1082 1083 if (cmd_dir == DMA_FROM_DEVICE) { 1084 data_direction = UTP_DEVICE_TO_HOST; 1085 *upiu_flags = UPIU_CMD_FLAGS_READ; 1086 } else if (cmd_dir == DMA_TO_DEVICE) { 1087 data_direction = UTP_HOST_TO_DEVICE; 1088 *upiu_flags = UPIU_CMD_FLAGS_WRITE; 1089 } else { 1090 data_direction = UTP_NO_DATA_TRANSFER; 1091 *upiu_flags = UPIU_CMD_FLAGS_NONE; 1092 } 1093 1094 dword_0 = data_direction | (lrbp->command_type 1095 << UPIU_COMMAND_TYPE_OFFSET); 1096 if (lrbp->intr_cmd) 1097 dword_0 |= UTP_REQ_DESC_INT_CMD; 1098 1099 /* Transfer request descriptor header fields */ 1100 req_desc->header.dword_0 = cpu_to_le32(dword_0); 1101 1102 /* 1103 * assigning invalid value for command status. Controller 1104 * updates OCS on command completion, with the command 1105 * status 1106 */ 1107 req_desc->header.dword_2 = 1108 cpu_to_le32(OCS_INVALID_COMMAND_STATUS); 1109} 1110 1111/** 1112 * ufshcd_prepare_utp_scsi_cmd_upiu() - fills the utp_transfer_req_desc, 1113 * for scsi commands 1114 * @lrbp - local reference block pointer 1115 * @upiu_flags - flags 1116 */ 1117static 1118void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb *lrbp, u32 upiu_flags) 1119{ 1120 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr; 1121 1122 /* command descriptor fields */ 1123 ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD( 1124 UPIU_TRANSACTION_COMMAND, upiu_flags, 1125 lrbp->lun, lrbp->task_tag); 1126 ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD( 1127 UPIU_COMMAND_SET_TYPE_SCSI, 0, 0, 0); 1128 1129 /* Total EHS length and Data segment length will be zero */ 1130 ucd_req_ptr->header.dword_2 = 0; 1131 1132 ucd_req_ptr->sc.exp_data_transfer_len = 1133 cpu_to_be32(lrbp->cmd->sdb.length); 1134 1135 memcpy(ucd_req_ptr->sc.cdb, lrbp->cmd->cmnd, 1136 (min_t(unsigned short, lrbp->cmd->cmd_len, MAX_CDB_SIZE))); 1137} 1138 1139/** 1140 * ufshcd_prepare_utp_query_req_upiu() - fills the utp_transfer_req_desc, 1141 * for query requsts 1142 * @hba: UFS hba 1143 * @lrbp: local reference block pointer 1144 * @upiu_flags: flags 1145 */ 1146static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba *hba, 1147 struct ufshcd_lrb *lrbp, u32 upiu_flags) 1148{ 1149 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr; 1150 struct ufs_query *query = &hba->dev_cmd.query; 1151 u16 len = be16_to_cpu(query->request.upiu_req.length); 1152 u8 *descp = (u8 *)lrbp->ucd_req_ptr + GENERAL_UPIU_REQUEST_SIZE; 1153 1154 /* Query request header */ 1155 ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD( 1156 UPIU_TRANSACTION_QUERY_REQ, upiu_flags, 1157 lrbp->lun, lrbp->task_tag); 1158 ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD( 1159 0, query->request.query_func, 0, 0); 1160 1161 /* Data segment length */ 1162 ucd_req_ptr->header.dword_2 = UPIU_HEADER_DWORD( 1163 0, 0, len >> 8, (u8)len); 1164 1165 /* Copy the Query Request buffer as is */ 1166 memcpy(&ucd_req_ptr->qr, &query->request.upiu_req, 1167 QUERY_OSF_SIZE); 1168 1169 /* Copy the Descriptor */ 1170 if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC) 1171 memcpy(descp, query->descriptor, len); 1172 1173} 1174 1175static inline void ufshcd_prepare_utp_nop_upiu(struct ufshcd_lrb *lrbp) 1176{ 1177 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr; 1178 1179 memset(ucd_req_ptr, 0, sizeof(struct utp_upiu_req)); 1180 1181 /* command descriptor fields */ 1182 ucd_req_ptr->header.dword_0 = 1183 UPIU_HEADER_DWORD( 1184 UPIU_TRANSACTION_NOP_OUT, 0, 0, lrbp->task_tag); 1185} 1186 1187/** 1188 * ufshcd_compose_upiu - form UFS Protocol Information Unit(UPIU) 1189 * @hba - per adapter instance 1190 * @lrb - pointer to local reference block 1191 */ 1192static int ufshcd_compose_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) 1193{ 1194 u32 upiu_flags; 1195 int ret = 0; 1196 1197 switch (lrbp->command_type) { 1198 case UTP_CMD_TYPE_SCSI: 1199 if (likely(lrbp->cmd)) { 1200 ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, 1201 lrbp->cmd->sc_data_direction); 1202 ufshcd_prepare_utp_scsi_cmd_upiu(lrbp, upiu_flags); 1203 } else { 1204 ret = -EINVAL; 1205 } 1206 break; 1207 case UTP_CMD_TYPE_DEV_MANAGE: 1208 ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, DMA_NONE); 1209 if (hba->dev_cmd.type == DEV_CMD_TYPE_QUERY) 1210 ufshcd_prepare_utp_query_req_upiu( 1211 hba, lrbp, upiu_flags); 1212 else if (hba->dev_cmd.type == DEV_CMD_TYPE_NOP) 1213 ufshcd_prepare_utp_nop_upiu(lrbp); 1214 else 1215 ret = -EINVAL; 1216 break; 1217 case UTP_CMD_TYPE_UFS: 1218 /* For UFS native command implementation */ 1219 ret = -ENOTSUPP; 1220 dev_err(hba->dev, "%s: UFS native command are not supported\n", 1221 __func__); 1222 break; 1223 default: 1224 ret = -ENOTSUPP; 1225 dev_err(hba->dev, "%s: unknown command type: 0x%x\n", 1226 __func__, lrbp->command_type); 1227 break; 1228 } /* end of switch */ 1229 1230 return ret; 1231} 1232 1233/* 1234 * ufshcd_scsi_to_upiu_lun - maps scsi LUN to UPIU LUN 1235 * @scsi_lun: scsi LUN id 1236 * 1237 * Returns UPIU LUN id 1238 */ 1239static inline u8 ufshcd_scsi_to_upiu_lun(unsigned int scsi_lun) 1240{ 1241 if (scsi_is_wlun(scsi_lun)) 1242 return (scsi_lun & UFS_UPIU_MAX_UNIT_NUM_ID) 1243 | UFS_UPIU_WLUN_ID; 1244 else 1245 return scsi_lun & UFS_UPIU_MAX_UNIT_NUM_ID; 1246} 1247 1248/** 1249 * ufshcd_upiu_wlun_to_scsi_wlun - maps UPIU W-LUN id to SCSI W-LUN ID 1250 * @scsi_lun: UPIU W-LUN id 1251 * 1252 * Returns SCSI W-LUN id 1253 */ 1254static inline u16 ufshcd_upiu_wlun_to_scsi_wlun(u8 upiu_wlun_id) 1255{ 1256 return (upiu_wlun_id & ~UFS_UPIU_WLUN_ID) | SCSI_W_LUN_BASE; 1257} 1258 1259/** 1260 * ufshcd_queuecommand - main entry point for SCSI requests 1261 * @cmd: command from SCSI Midlayer 1262 * @done: call back function 1263 * 1264 * Returns 0 for success, non-zero in case of failure 1265 */ 1266static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd) 1267{ 1268 struct ufshcd_lrb *lrbp; 1269 struct ufs_hba *hba; 1270 unsigned long flags; 1271 int tag; 1272 int err = 0; 1273 1274 hba = shost_priv(host); 1275 1276 tag = cmd->request->tag; 1277 1278 spin_lock_irqsave(hba->host->host_lock, flags); 1279 switch (hba->ufshcd_state) { 1280 case UFSHCD_STATE_OPERATIONAL: 1281 break; 1282 case UFSHCD_STATE_RESET: 1283 err = SCSI_MLQUEUE_HOST_BUSY; 1284 goto out_unlock; 1285 case UFSHCD_STATE_ERROR: 1286 set_host_byte(cmd, DID_ERROR); 1287 cmd->scsi_done(cmd); 1288 goto out_unlock; 1289 default: 1290 dev_WARN_ONCE(hba->dev, 1, "%s: invalid state %d\n", 1291 __func__, hba->ufshcd_state); 1292 set_host_byte(cmd, DID_BAD_TARGET); 1293 cmd->scsi_done(cmd); 1294 goto out_unlock; 1295 } 1296 spin_unlock_irqrestore(hba->host->host_lock, flags); 1297 1298 /* acquire the tag to make sure device cmds don't use it */ 1299 if (test_and_set_bit_lock(tag, &hba->lrb_in_use)) { 1300 /* 1301 * Dev manage command in progress, requeue the command. 1302 * Requeuing the command helps in cases where the request *may* 1303 * find different tag instead of waiting for dev manage command 1304 * completion. 1305 */ 1306 err = SCSI_MLQUEUE_HOST_BUSY; 1307 goto out; 1308 } 1309 1310 err = ufshcd_hold(hba, true); 1311 if (err) { 1312 err = SCSI_MLQUEUE_HOST_BUSY; 1313 clear_bit_unlock(tag, &hba->lrb_in_use); 1314 goto out; 1315 } 1316 WARN_ON(hba->clk_gating.state != CLKS_ON); 1317 1318 lrbp = &hba->lrb[tag]; 1319 1320 WARN_ON(lrbp->cmd); 1321 lrbp->cmd = cmd; 1322 lrbp->sense_bufflen = SCSI_SENSE_BUFFERSIZE; 1323 lrbp->sense_buffer = cmd->sense_buffer; 1324 lrbp->task_tag = tag; 1325 lrbp->lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun); 1326 lrbp->intr_cmd = false; 1327 lrbp->command_type = UTP_CMD_TYPE_SCSI; 1328 1329 /* form UPIU before issuing the command */ 1330 ufshcd_compose_upiu(hba, lrbp); 1331 err = ufshcd_map_sg(lrbp); 1332 if (err) { 1333 lrbp->cmd = NULL; 1334 clear_bit_unlock(tag, &hba->lrb_in_use); 1335 goto out; 1336 } 1337 1338 /* issue command to the controller */ 1339 spin_lock_irqsave(hba->host->host_lock, flags); 1340 ufshcd_send_command(hba, tag); 1341out_unlock: 1342 spin_unlock_irqrestore(hba->host->host_lock, flags); 1343out: 1344 return err; 1345} 1346 1347static int ufshcd_compose_dev_cmd(struct ufs_hba *hba, 1348 struct ufshcd_lrb *lrbp, enum dev_cmd_type cmd_type, int tag) 1349{ 1350 lrbp->cmd = NULL; 1351 lrbp->sense_bufflen = 0; 1352 lrbp->sense_buffer = NULL; 1353 lrbp->task_tag = tag; 1354 lrbp->lun = 0; /* device management cmd is not specific to any LUN */ 1355 lrbp->command_type = UTP_CMD_TYPE_DEV_MANAGE; 1356 lrbp->intr_cmd = true; /* No interrupt aggregation */ 1357 hba->dev_cmd.type = cmd_type; 1358 1359 return ufshcd_compose_upiu(hba, lrbp); 1360} 1361 1362static int 1363ufshcd_clear_cmd(struct ufs_hba *hba, int tag) 1364{ 1365 int err = 0; 1366 unsigned long flags; 1367 u32 mask = 1 << tag; 1368 1369 /* clear outstanding transaction before retry */ 1370 spin_lock_irqsave(hba->host->host_lock, flags); 1371 ufshcd_utrl_clear(hba, tag); 1372 spin_unlock_irqrestore(hba->host->host_lock, flags); 1373 1374 /* 1375 * wait for for h/w to clear corresponding bit in door-bell. 1376 * max. wait is 1 sec. 1377 */ 1378 err = ufshcd_wait_for_register(hba, 1379 REG_UTP_TRANSFER_REQ_DOOR_BELL, 1380 mask, ~mask, 1000, 1000); 1381 1382 return err; 1383} 1384 1385static int 1386ufshcd_check_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) 1387{ 1388 struct ufs_query_res *query_res = &hba->dev_cmd.query.response; 1389 1390 /* Get the UPIU response */ 1391 query_res->response = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr) >> 1392 UPIU_RSP_CODE_OFFSET; 1393 return query_res->response; 1394} 1395 1396/** 1397 * ufshcd_dev_cmd_completion() - handles device management command responses 1398 * @hba: per adapter instance 1399 * @lrbp: pointer to local reference block 1400 */ 1401static int 1402ufshcd_dev_cmd_completion(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) 1403{ 1404 int resp; 1405 int err = 0; 1406 1407 resp = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr); 1408 1409 switch (resp) { 1410 case UPIU_TRANSACTION_NOP_IN: 1411 if (hba->dev_cmd.type != DEV_CMD_TYPE_NOP) { 1412 err = -EINVAL; 1413 dev_err(hba->dev, "%s: unexpected response %x\n", 1414 __func__, resp); 1415 } 1416 break; 1417 case UPIU_TRANSACTION_QUERY_RSP: 1418 err = ufshcd_check_query_response(hba, lrbp); 1419 if (!err) 1420 err = ufshcd_copy_query_response(hba, lrbp); 1421 break; 1422 case UPIU_TRANSACTION_REJECT_UPIU: 1423 /* TODO: handle Reject UPIU Response */ 1424 err = -EPERM; 1425 dev_err(hba->dev, "%s: Reject UPIU not fully implemented\n", 1426 __func__); 1427 break; 1428 default: 1429 err = -EINVAL; 1430 dev_err(hba->dev, "%s: Invalid device management cmd response: %x\n", 1431 __func__, resp); 1432 break; 1433 } 1434 1435 return err; 1436} 1437 1438static int ufshcd_wait_for_dev_cmd(struct ufs_hba *hba, 1439 struct ufshcd_lrb *lrbp, int max_timeout) 1440{ 1441 int err = 0; 1442 unsigned long time_left; 1443 unsigned long flags; 1444 1445 time_left = wait_for_completion_timeout(hba->dev_cmd.complete, 1446 msecs_to_jiffies(max_timeout)); 1447 1448 spin_lock_irqsave(hba->host->host_lock, flags); 1449 hba->dev_cmd.complete = NULL; 1450 if (likely(time_left)) { 1451 err = ufshcd_get_tr_ocs(lrbp); 1452 if (!err) 1453 err = ufshcd_dev_cmd_completion(hba, lrbp); 1454 } 1455 spin_unlock_irqrestore(hba->host->host_lock, flags); 1456 1457 if (!time_left) { 1458 err = -ETIMEDOUT; 1459 if (!ufshcd_clear_cmd(hba, lrbp->task_tag)) 1460 /* sucessfully cleared the command, retry if needed */ 1461 err = -EAGAIN; 1462 } 1463 1464 return err; 1465} 1466 1467/** 1468 * ufshcd_get_dev_cmd_tag - Get device management command tag 1469 * @hba: per-adapter instance 1470 * @tag: pointer to variable with available slot value 1471 * 1472 * Get a free slot and lock it until device management command 1473 * completes. 1474 * 1475 * Returns false if free slot is unavailable for locking, else 1476 * return true with tag value in @tag. 1477 */ 1478static bool ufshcd_get_dev_cmd_tag(struct ufs_hba *hba, int *tag_out) 1479{ 1480 int tag; 1481 bool ret = false; 1482 unsigned long tmp; 1483 1484 if (!tag_out) 1485 goto out; 1486 1487 do { 1488 tmp = ~hba->lrb_in_use; 1489 tag = find_last_bit(&tmp, hba->nutrs); 1490 if (tag >= hba->nutrs) 1491 goto out; 1492 } while (test_and_set_bit_lock(tag, &hba->lrb_in_use)); 1493 1494 *tag_out = tag; 1495 ret = true; 1496out: 1497 return ret; 1498} 1499 1500static inline void ufshcd_put_dev_cmd_tag(struct ufs_hba *hba, int tag) 1501{ 1502 clear_bit_unlock(tag, &hba->lrb_in_use); 1503} 1504 1505/** 1506 * ufshcd_exec_dev_cmd - API for sending device management requests 1507 * @hba - UFS hba 1508 * @cmd_type - specifies the type (NOP, Query...) 1509 * @timeout - time in seconds 1510 * 1511 * NOTE: Since there is only one available tag for device management commands, 1512 * it is expected you hold the hba->dev_cmd.lock mutex. 1513 */ 1514static int ufshcd_exec_dev_cmd(struct ufs_hba *hba, 1515 enum dev_cmd_type cmd_type, int timeout) 1516{ 1517 struct ufshcd_lrb *lrbp; 1518 int err; 1519 int tag; 1520 struct completion wait; 1521 unsigned long flags; 1522 1523 /* 1524 * Get free slot, sleep if slots are unavailable. 1525 * Even though we use wait_event() which sleeps indefinitely, 1526 * the maximum wait time is bounded by SCSI request timeout. 1527 */ 1528 wait_event(hba->dev_cmd.tag_wq, ufshcd_get_dev_cmd_tag(hba, &tag)); 1529 1530 init_completion(&wait); 1531 lrbp = &hba->lrb[tag]; 1532 WARN_ON(lrbp->cmd); 1533 err = ufshcd_compose_dev_cmd(hba, lrbp, cmd_type, tag); 1534 if (unlikely(err)) 1535 goto out_put_tag; 1536 1537 hba->dev_cmd.complete = &wait; 1538 1539 spin_lock_irqsave(hba->host->host_lock, flags); 1540 ufshcd_send_command(hba, tag); 1541 spin_unlock_irqrestore(hba->host->host_lock, flags); 1542 1543 err = ufshcd_wait_for_dev_cmd(hba, lrbp, timeout); 1544 1545out_put_tag: 1546 ufshcd_put_dev_cmd_tag(hba, tag); 1547 wake_up(&hba->dev_cmd.tag_wq); 1548 return err; 1549} 1550 1551/** 1552 * ufshcd_init_query() - init the query response and request parameters 1553 * @hba: per-adapter instance 1554 * @request: address of the request pointer to be initialized 1555 * @response: address of the response pointer to be initialized 1556 * @opcode: operation to perform 1557 * @idn: flag idn to access 1558 * @index: LU number to access 1559 * @selector: query/flag/descriptor further identification 1560 */ 1561static inline void ufshcd_init_query(struct ufs_hba *hba, 1562 struct ufs_query_req **request, struct ufs_query_res **response, 1563 enum query_opcode opcode, u8 idn, u8 index, u8 selector) 1564{ 1565 *request = &hba->dev_cmd.query.request; 1566 *response = &hba->dev_cmd.query.response; 1567 memset(*request, 0, sizeof(struct ufs_query_req)); 1568 memset(*response, 0, sizeof(struct ufs_query_res)); 1569 (*request)->upiu_req.opcode = opcode; 1570 (*request)->upiu_req.idn = idn; 1571 (*request)->upiu_req.index = index; 1572 (*request)->upiu_req.selector = selector; 1573} 1574 1575/** 1576 * ufshcd_query_flag() - API function for sending flag query requests 1577 * hba: per-adapter instance 1578 * query_opcode: flag query to perform 1579 * idn: flag idn to access 1580 * flag_res: the flag value after the query request completes 1581 * 1582 * Returns 0 for success, non-zero in case of failure 1583 */ 1584static int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode, 1585 enum flag_idn idn, bool *flag_res) 1586{ 1587 struct ufs_query_req *request = NULL; 1588 struct ufs_query_res *response = NULL; 1589 int err, index = 0, selector = 0; 1590 1591 BUG_ON(!hba); 1592 1593 ufshcd_hold(hba, false); 1594 mutex_lock(&hba->dev_cmd.lock); 1595 ufshcd_init_query(hba, &request, &response, opcode, idn, index, 1596 selector); 1597 1598 switch (opcode) { 1599 case UPIU_QUERY_OPCODE_SET_FLAG: 1600 case UPIU_QUERY_OPCODE_CLEAR_FLAG: 1601 case UPIU_QUERY_OPCODE_TOGGLE_FLAG: 1602 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST; 1603 break; 1604 case UPIU_QUERY_OPCODE_READ_FLAG: 1605 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST; 1606 if (!flag_res) { 1607 /* No dummy reads */ 1608 dev_err(hba->dev, "%s: Invalid argument for read request\n", 1609 __func__); 1610 err = -EINVAL; 1611 goto out_unlock; 1612 } 1613 break; 1614 default: 1615 dev_err(hba->dev, 1616 "%s: Expected query flag opcode but got = %d\n", 1617 __func__, opcode); 1618 err = -EINVAL; 1619 goto out_unlock; 1620 } 1621 1622 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT); 1623 1624 if (err) { 1625 dev_err(hba->dev, 1626 "%s: Sending flag query for idn %d failed, err = %d\n", 1627 __func__, idn, err); 1628 goto out_unlock; 1629 } 1630 1631 if (flag_res) 1632 *flag_res = (be32_to_cpu(response->upiu_res.value) & 1633 MASK_QUERY_UPIU_FLAG_LOC) & 0x1; 1634 1635out_unlock: 1636 mutex_unlock(&hba->dev_cmd.lock); 1637 ufshcd_release(hba); 1638 return err; 1639} 1640 1641/** 1642 * ufshcd_query_attr - API function for sending attribute requests 1643 * hba: per-adapter instance 1644 * opcode: attribute opcode 1645 * idn: attribute idn to access 1646 * index: index field 1647 * selector: selector field 1648 * attr_val: the attribute value after the query request completes 1649 * 1650 * Returns 0 for success, non-zero in case of failure 1651*/ 1652static int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode, 1653 enum attr_idn idn, u8 index, u8 selector, u32 *attr_val) 1654{ 1655 struct ufs_query_req *request = NULL; 1656 struct ufs_query_res *response = NULL; 1657 int err; 1658 1659 BUG_ON(!hba); 1660 1661 ufshcd_hold(hba, false); 1662 if (!attr_val) { 1663 dev_err(hba->dev, "%s: attribute value required for opcode 0x%x\n", 1664 __func__, opcode); 1665 err = -EINVAL; 1666 goto out; 1667 } 1668 1669 mutex_lock(&hba->dev_cmd.lock); 1670 ufshcd_init_query(hba, &request, &response, opcode, idn, index, 1671 selector); 1672 1673 switch (opcode) { 1674 case UPIU_QUERY_OPCODE_WRITE_ATTR: 1675 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST; 1676 request->upiu_req.value = cpu_to_be32(*attr_val); 1677 break; 1678 case UPIU_QUERY_OPCODE_READ_ATTR: 1679 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST; 1680 break; 1681 default: 1682 dev_err(hba->dev, "%s: Expected query attr opcode but got = 0x%.2x\n", 1683 __func__, opcode); 1684 err = -EINVAL; 1685 goto out_unlock; 1686 } 1687 1688 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT); 1689 1690 if (err) { 1691 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, err = %d\n", 1692 __func__, opcode, idn, err); 1693 goto out_unlock; 1694 } 1695 1696 *attr_val = be32_to_cpu(response->upiu_res.value); 1697 1698out_unlock: 1699 mutex_unlock(&hba->dev_cmd.lock); 1700out: 1701 ufshcd_release(hba); 1702 return err; 1703} 1704 1705/** 1706 * ufshcd_query_descriptor - API function for sending descriptor requests 1707 * hba: per-adapter instance 1708 * opcode: attribute opcode 1709 * idn: attribute idn to access 1710 * index: index field 1711 * selector: selector field 1712 * desc_buf: the buffer that contains the descriptor 1713 * buf_len: length parameter passed to the device 1714 * 1715 * Returns 0 for success, non-zero in case of failure. 1716 * The buf_len parameter will contain, on return, the length parameter 1717 * received on the response. 1718 */ 1719static int ufshcd_query_descriptor(struct ufs_hba *hba, 1720 enum query_opcode opcode, enum desc_idn idn, u8 index, 1721 u8 selector, u8 *desc_buf, int *buf_len) 1722{ 1723 struct ufs_query_req *request = NULL; 1724 struct ufs_query_res *response = NULL; 1725 int err; 1726 1727 BUG_ON(!hba); 1728 1729 ufshcd_hold(hba, false); 1730 if (!desc_buf) { 1731 dev_err(hba->dev, "%s: descriptor buffer required for opcode 0x%x\n", 1732 __func__, opcode); 1733 err = -EINVAL; 1734 goto out; 1735 } 1736 1737 if (*buf_len <= QUERY_DESC_MIN_SIZE || *buf_len > QUERY_DESC_MAX_SIZE) { 1738 dev_err(hba->dev, "%s: descriptor buffer size (%d) is out of range\n", 1739 __func__, *buf_len); 1740 err = -EINVAL; 1741 goto out; 1742 } 1743 1744 mutex_lock(&hba->dev_cmd.lock); 1745 ufshcd_init_query(hba, &request, &response, opcode, idn, index, 1746 selector); 1747 hba->dev_cmd.query.descriptor = desc_buf; 1748 request->upiu_req.length = cpu_to_be16(*buf_len); 1749 1750 switch (opcode) { 1751 case UPIU_QUERY_OPCODE_WRITE_DESC: 1752 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST; 1753 break; 1754 case UPIU_QUERY_OPCODE_READ_DESC: 1755 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST; 1756 break; 1757 default: 1758 dev_err(hba->dev, 1759 "%s: Expected query descriptor opcode but got = 0x%.2x\n", 1760 __func__, opcode); 1761 err = -EINVAL; 1762 goto out_unlock; 1763 } 1764 1765 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT); 1766 1767 if (err) { 1768 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, err = %d\n", 1769 __func__, opcode, idn, err); 1770 goto out_unlock; 1771 } 1772 1773 hba->dev_cmd.query.descriptor = NULL; 1774 *buf_len = be16_to_cpu(response->upiu_res.length); 1775 1776out_unlock: 1777 mutex_unlock(&hba->dev_cmd.lock); 1778out: 1779 ufshcd_release(hba); 1780 return err; 1781} 1782 1783/** 1784 * ufshcd_read_desc_param - read the specified descriptor parameter 1785 * @hba: Pointer to adapter instance 1786 * @desc_id: descriptor idn value 1787 * @desc_index: descriptor index 1788 * @param_offset: offset of the parameter to read 1789 * @param_read_buf: pointer to buffer where parameter would be read 1790 * @param_size: sizeof(param_read_buf) 1791 * 1792 * Return 0 in case of success, non-zero otherwise 1793 */ 1794static int ufshcd_read_desc_param(struct ufs_hba *hba, 1795 enum desc_idn desc_id, 1796 int desc_index, 1797 u32 param_offset, 1798 u8 *param_read_buf, 1799 u32 param_size) 1800{ 1801 int ret; 1802 u8 *desc_buf; 1803 u32 buff_len; 1804 bool is_kmalloc = true; 1805 1806 /* safety checks */ 1807 if (desc_id >= QUERY_DESC_IDN_MAX) 1808 return -EINVAL; 1809 1810 buff_len = ufs_query_desc_max_size[desc_id]; 1811 if ((param_offset + param_size) > buff_len) 1812 return -EINVAL; 1813 1814 if (!param_offset && (param_size == buff_len)) { 1815 /* memory space already available to hold full descriptor */ 1816 desc_buf = param_read_buf; 1817 is_kmalloc = false; 1818 } else { 1819 /* allocate memory to hold full descriptor */ 1820 desc_buf = kmalloc(buff_len, GFP_KERNEL); 1821 if (!desc_buf) 1822 return -ENOMEM; 1823 } 1824 1825 ret = ufshcd_query_descriptor(hba, UPIU_QUERY_OPCODE_READ_DESC, 1826 desc_id, desc_index, 0, desc_buf, 1827 &buff_len); 1828 1829 if (ret || (buff_len < ufs_query_desc_max_size[desc_id]) || 1830 (desc_buf[QUERY_DESC_LENGTH_OFFSET] != 1831 ufs_query_desc_max_size[desc_id]) 1832 || (desc_buf[QUERY_DESC_DESC_TYPE_OFFSET] != desc_id)) { 1833 dev_err(hba->dev, "%s: Failed reading descriptor. desc_id %d param_offset %d buff_len %d ret %d", 1834 __func__, desc_id, param_offset, buff_len, ret); 1835 if (!ret) 1836 ret = -EINVAL; 1837 1838 goto out; 1839 } 1840 1841 if (is_kmalloc) 1842 memcpy(param_read_buf, &desc_buf[param_offset], param_size); 1843out: 1844 if (is_kmalloc) 1845 kfree(desc_buf); 1846 return ret; 1847} 1848 1849static inline int ufshcd_read_desc(struct ufs_hba *hba, 1850 enum desc_idn desc_id, 1851 int desc_index, 1852 u8 *buf, 1853 u32 size) 1854{ 1855 return ufshcd_read_desc_param(hba, desc_id, desc_index, 0, buf, size); 1856} 1857 1858static inline int ufshcd_read_power_desc(struct ufs_hba *hba, 1859 u8 *buf, 1860 u32 size) 1861{ 1862 return ufshcd_read_desc(hba, QUERY_DESC_IDN_POWER, 0, buf, size); 1863} 1864 1865/** 1866 * ufshcd_read_unit_desc_param - read the specified unit descriptor parameter 1867 * @hba: Pointer to adapter instance 1868 * @lun: lun id 1869 * @param_offset: offset of the parameter to read 1870 * @param_read_buf: pointer to buffer where parameter would be read 1871 * @param_size: sizeof(param_read_buf) 1872 * 1873 * Return 0 in case of success, non-zero otherwise 1874 */ 1875static inline int ufshcd_read_unit_desc_param(struct ufs_hba *hba, 1876 int lun, 1877 enum unit_desc_param param_offset, 1878 u8 *param_read_buf, 1879 u32 param_size) 1880{ 1881 /* 1882 * Unit descriptors are only available for general purpose LUs (LUN id 1883 * from 0 to 7) and RPMB Well known LU. 1884 */ 1885 if (lun != UFS_UPIU_RPMB_WLUN && (lun >= UFS_UPIU_MAX_GENERAL_LUN)) 1886 return -EOPNOTSUPP; 1887 1888 return ufshcd_read_desc_param(hba, QUERY_DESC_IDN_UNIT, lun, 1889 param_offset, param_read_buf, param_size); 1890} 1891 1892/** 1893 * ufshcd_memory_alloc - allocate memory for host memory space data structures 1894 * @hba: per adapter instance 1895 * 1896 * 1. Allocate DMA memory for Command Descriptor array 1897 * Each command descriptor consist of Command UPIU, Response UPIU and PRDT 1898 * 2. Allocate DMA memory for UTP Transfer Request Descriptor List (UTRDL). 1899 * 3. Allocate DMA memory for UTP Task Management Request Descriptor List 1900 * (UTMRDL) 1901 * 4. Allocate memory for local reference block(lrb). 1902 * 1903 * Returns 0 for success, non-zero in case of failure 1904 */ 1905static int ufshcd_memory_alloc(struct ufs_hba *hba) 1906{ 1907 size_t utmrdl_size, utrdl_size, ucdl_size; 1908 1909 /* Allocate memory for UTP command descriptors */ 1910 ucdl_size = (sizeof(struct utp_transfer_cmd_desc) * hba->nutrs); 1911 hba->ucdl_base_addr = dmam_alloc_coherent(hba->dev, 1912 ucdl_size, 1913 &hba->ucdl_dma_addr, 1914 GFP_KERNEL); 1915 1916 /* 1917 * UFSHCI requires UTP command descriptor to be 128 byte aligned. 1918 * make sure hba->ucdl_dma_addr is aligned to PAGE_SIZE 1919 * if hba->ucdl_dma_addr is aligned to PAGE_SIZE, then it will 1920 * be aligned to 128 bytes as well 1921 */ 1922 if (!hba->ucdl_base_addr || 1923 WARN_ON(hba->ucdl_dma_addr & (PAGE_SIZE - 1))) { 1924 dev_err(hba->dev, 1925 "Command Descriptor Memory allocation failed\n"); 1926 goto out; 1927 } 1928 1929 /* 1930 * Allocate memory for UTP Transfer descriptors 1931 * UFSHCI requires 1024 byte alignment of UTRD 1932 */ 1933 utrdl_size = (sizeof(struct utp_transfer_req_desc) * hba->nutrs); 1934 hba->utrdl_base_addr = dmam_alloc_coherent(hba->dev, 1935 utrdl_size, 1936 &hba->utrdl_dma_addr, 1937 GFP_KERNEL); 1938 if (!hba->utrdl_base_addr || 1939 WARN_ON(hba->utrdl_dma_addr & (PAGE_SIZE - 1))) { 1940 dev_err(hba->dev, 1941 "Transfer Descriptor Memory allocation failed\n"); 1942 goto out; 1943 } 1944 1945 /* 1946 * Allocate memory for UTP Task Management descriptors 1947 * UFSHCI requires 1024 byte alignment of UTMRD 1948 */ 1949 utmrdl_size = sizeof(struct utp_task_req_desc) * hba->nutmrs; 1950 hba->utmrdl_base_addr = dmam_alloc_coherent(hba->dev, 1951 utmrdl_size, 1952 &hba->utmrdl_dma_addr, 1953 GFP_KERNEL); 1954 if (!hba->utmrdl_base_addr || 1955 WARN_ON(hba->utmrdl_dma_addr & (PAGE_SIZE - 1))) { 1956 dev_err(hba->dev, 1957 "Task Management Descriptor Memory allocation failed\n"); 1958 goto out; 1959 } 1960 1961 /* Allocate memory for local reference block */ 1962 hba->lrb = devm_kzalloc(hba->dev, 1963 hba->nutrs * sizeof(struct ufshcd_lrb), 1964 GFP_KERNEL); 1965 if (!hba->lrb) { 1966 dev_err(hba->dev, "LRB Memory allocation failed\n"); 1967 goto out; 1968 } 1969 return 0; 1970out: 1971 return -ENOMEM; 1972} 1973 1974/** 1975 * ufshcd_host_memory_configure - configure local reference block with 1976 * memory offsets 1977 * @hba: per adapter instance 1978 * 1979 * Configure Host memory space 1980 * 1. Update Corresponding UTRD.UCDBA and UTRD.UCDBAU with UCD DMA 1981 * address. 1982 * 2. Update each UTRD with Response UPIU offset, Response UPIU length 1983 * and PRDT offset. 1984 * 3. Save the corresponding addresses of UTRD, UCD.CMD, UCD.RSP and UCD.PRDT 1985 * into local reference block. 1986 */ 1987static void ufshcd_host_memory_configure(struct ufs_hba *hba) 1988{ 1989 struct utp_transfer_cmd_desc *cmd_descp; 1990 struct utp_transfer_req_desc *utrdlp; 1991 dma_addr_t cmd_desc_dma_addr; 1992 dma_addr_t cmd_desc_element_addr; 1993 u16 response_offset; 1994 u16 prdt_offset; 1995 int cmd_desc_size; 1996 int i; 1997 1998 utrdlp = hba->utrdl_base_addr; 1999 cmd_descp = hba->ucdl_base_addr; 2000 2001 response_offset = 2002 offsetof(struct utp_transfer_cmd_desc, response_upiu); 2003 prdt_offset = 2004 offsetof(struct utp_transfer_cmd_desc, prd_table); 2005 2006 cmd_desc_size = sizeof(struct utp_transfer_cmd_desc); 2007 cmd_desc_dma_addr = hba->ucdl_dma_addr; 2008 2009 for (i = 0; i < hba->nutrs; i++) { 2010 /* Configure UTRD with command descriptor base address */ 2011 cmd_desc_element_addr = 2012 (cmd_desc_dma_addr + (cmd_desc_size * i)); 2013 utrdlp[i].command_desc_base_addr_lo = 2014 cpu_to_le32(lower_32_bits(cmd_desc_element_addr)); 2015 utrdlp[i].command_desc_base_addr_hi = 2016 cpu_to_le32(upper_32_bits(cmd_desc_element_addr)); 2017 2018 /* Response upiu and prdt offset should be in double words */ 2019 utrdlp[i].response_upiu_offset = 2020 cpu_to_le16((response_offset >> 2)); 2021 utrdlp[i].prd_table_offset = 2022 cpu_to_le16((prdt_offset >> 2)); 2023 utrdlp[i].response_upiu_length = 2024 cpu_to_le16(ALIGNED_UPIU_SIZE >> 2); 2025 2026 hba->lrb[i].utr_descriptor_ptr = (utrdlp + i); 2027 hba->lrb[i].ucd_req_ptr = 2028 (struct utp_upiu_req *)(cmd_descp + i); 2029 hba->lrb[i].ucd_rsp_ptr = 2030 (struct utp_upiu_rsp *)cmd_descp[i].response_upiu; 2031 hba->lrb[i].ucd_prdt_ptr = 2032 (struct ufshcd_sg_entry *)cmd_descp[i].prd_table; 2033 } 2034} 2035 2036/** 2037 * ufshcd_dme_link_startup - Notify Unipro to perform link startup 2038 * @hba: per adapter instance 2039 * 2040 * UIC_CMD_DME_LINK_STARTUP command must be issued to Unipro layer, 2041 * in order to initialize the Unipro link startup procedure. 2042 * Once the Unipro links are up, the device connected to the controller 2043 * is detected. 2044 * 2045 * Returns 0 on success, non-zero value on failure 2046 */ 2047static int ufshcd_dme_link_startup(struct ufs_hba *hba) 2048{ 2049 struct uic_command uic_cmd = {0}; 2050 int ret; 2051 2052 uic_cmd.command = UIC_CMD_DME_LINK_STARTUP; 2053 2054 ret = ufshcd_send_uic_cmd(hba, &uic_cmd); 2055 if (ret) 2056 dev_err(hba->dev, 2057 "dme-link-startup: error code %d\n", ret); 2058 return ret; 2059} 2060 2061/** 2062 * ufshcd_dme_set_attr - UIC command for DME_SET, DME_PEER_SET 2063 * @hba: per adapter instance 2064 * @attr_sel: uic command argument1 2065 * @attr_set: attribute set type as uic command argument2 2066 * @mib_val: setting value as uic command argument3 2067 * @peer: indicate whether peer or local 2068 * 2069 * Returns 0 on success, non-zero value on failure 2070 */ 2071int ufshcd_dme_set_attr(struct ufs_hba *hba, u32 attr_sel, 2072 u8 attr_set, u32 mib_val, u8 peer) 2073{ 2074 struct uic_command uic_cmd = {0}; 2075 static const char *const action[] = { 2076 "dme-set", 2077 "dme-peer-set" 2078 }; 2079 const char *set = action[!!peer]; 2080 int ret; 2081 2082 uic_cmd.command = peer ? 2083 UIC_CMD_DME_PEER_SET : UIC_CMD_DME_SET; 2084 uic_cmd.argument1 = attr_sel; 2085 uic_cmd.argument2 = UIC_ARG_ATTR_TYPE(attr_set); 2086 uic_cmd.argument3 = mib_val; 2087 2088 ret = ufshcd_send_uic_cmd(hba, &uic_cmd); 2089 if (ret) 2090 dev_err(hba->dev, "%s: attr-id 0x%x val 0x%x error code %d\n", 2091 set, UIC_GET_ATTR_ID(attr_sel), mib_val, ret); 2092 2093 return ret; 2094} 2095EXPORT_SYMBOL_GPL(ufshcd_dme_set_attr); 2096 2097/** 2098 * ufshcd_dme_get_attr - UIC command for DME_GET, DME_PEER_GET 2099 * @hba: per adapter instance 2100 * @attr_sel: uic command argument1 2101 * @mib_val: the value of the attribute as returned by the UIC command 2102 * @peer: indicate whether peer or local 2103 * 2104 * Returns 0 on success, non-zero value on failure 2105 */ 2106int ufshcd_dme_get_attr(struct ufs_hba *hba, u32 attr_sel, 2107 u32 *mib_val, u8 peer) 2108{ 2109 struct uic_command uic_cmd = {0}; 2110 static const char *const action[] = { 2111 "dme-get", 2112 "dme-peer-get" 2113 }; 2114 const char *get = action[!!peer]; 2115 int ret; 2116 2117 uic_cmd.command = peer ? 2118 UIC_CMD_DME_PEER_GET : UIC_CMD_DME_GET; 2119 uic_cmd.argument1 = attr_sel; 2120 2121 ret = ufshcd_send_uic_cmd(hba, &uic_cmd); 2122 if (ret) { 2123 dev_err(hba->dev, "%s: attr-id 0x%x error code %d\n", 2124 get, UIC_GET_ATTR_ID(attr_sel), ret); 2125 goto out; 2126 } 2127 2128 if (mib_val) 2129 *mib_val = uic_cmd.argument3; 2130out: 2131 return ret; 2132} 2133EXPORT_SYMBOL_GPL(ufshcd_dme_get_attr); 2134 2135/** 2136 * ufshcd_uic_pwr_ctrl - executes UIC commands (which affects the link power 2137 * state) and waits for it to take effect. 2138 * 2139 * @hba: per adapter instance 2140 * @cmd: UIC command to execute 2141 * 2142 * DME operations like DME_SET(PA_PWRMODE), DME_HIBERNATE_ENTER & 2143 * DME_HIBERNATE_EXIT commands take some time to take its effect on both host 2144 * and device UniPro link and hence it's final completion would be indicated by 2145 * dedicated status bits in Interrupt Status register (UPMS, UHES, UHXS) in 2146 * addition to normal UIC command completion Status (UCCS). This function only 2147 * returns after the relevant status bits indicate the completion. 2148 * 2149 * Returns 0 on success, non-zero value on failure 2150 */ 2151static int ufshcd_uic_pwr_ctrl(struct ufs_hba *hba, struct uic_command *cmd) 2152{ 2153 struct completion uic_async_done; 2154 unsigned long flags; 2155 u8 status; 2156 int ret; 2157 2158 mutex_lock(&hba->uic_cmd_mutex); 2159 init_completion(&uic_async_done); 2160 2161 spin_lock_irqsave(hba->host->host_lock, flags); 2162 hba->uic_async_done = &uic_async_done; 2163 ret = __ufshcd_send_uic_cmd(hba, cmd); 2164 spin_unlock_irqrestore(hba->host->host_lock, flags); 2165 if (ret) { 2166 dev_err(hba->dev, 2167 "pwr ctrl cmd 0x%x with mode 0x%x uic error %d\n", 2168 cmd->command, cmd->argument3, ret); 2169 goto out; 2170 } 2171 ret = ufshcd_wait_for_uic_cmd(hba, cmd); 2172 if (ret) { 2173 dev_err(hba->dev, 2174 "pwr ctrl cmd 0x%x with mode 0x%x uic error %d\n", 2175 cmd->command, cmd->argument3, ret); 2176 goto out; 2177 } 2178 2179 if (!wait_for_completion_timeout(hba->uic_async_done, 2180 msecs_to_jiffies(UIC_CMD_TIMEOUT))) { 2181 dev_err(hba->dev, 2182 "pwr ctrl cmd 0x%x with mode 0x%x completion timeout\n", 2183 cmd->command, cmd->argument3); 2184 ret = -ETIMEDOUT; 2185 goto out; 2186 } 2187 2188 status = ufshcd_get_upmcrs(hba); 2189 if (status != PWR_LOCAL) { 2190 dev_err(hba->dev, 2191 "pwr ctrl cmd 0x%0x failed, host umpcrs:0x%x\n", 2192 cmd->command, status); 2193 ret = (status != PWR_OK) ? status : -1; 2194 } 2195out: 2196 spin_lock_irqsave(hba->host->host_lock, flags); 2197 hba->uic_async_done = NULL; 2198 spin_unlock_irqrestore(hba->host->host_lock, flags); 2199 mutex_unlock(&hba->uic_cmd_mutex); 2200 2201 return ret; 2202} 2203 2204/** 2205 * ufshcd_uic_change_pwr_mode - Perform the UIC power mode chage 2206 * using DME_SET primitives. 2207 * @hba: per adapter instance 2208 * @mode: powr mode value 2209 * 2210 * Returns 0 on success, non-zero value on failure 2211 */ 2212static int ufshcd_uic_change_pwr_mode(struct ufs_hba *hba, u8 mode) 2213{ 2214 struct uic_command uic_cmd = {0}; 2215 int ret; 2216 2217 uic_cmd.command = UIC_CMD_DME_SET; 2218 uic_cmd.argument1 = UIC_ARG_MIB(PA_PWRMODE); 2219 uic_cmd.argument3 = mode; 2220 ufshcd_hold(hba, false); 2221 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd); 2222 ufshcd_release(hba); 2223 2224 return ret; 2225} 2226 2227static int ufshcd_uic_hibern8_enter(struct ufs_hba *hba) 2228{ 2229 struct uic_command uic_cmd = {0}; 2230 2231 uic_cmd.command = UIC_CMD_DME_HIBER_ENTER; 2232 2233 return ufshcd_uic_pwr_ctrl(hba, &uic_cmd); 2234} 2235 2236static int ufshcd_uic_hibern8_exit(struct ufs_hba *hba) 2237{ 2238 struct uic_command uic_cmd = {0}; 2239 int ret; 2240 2241 uic_cmd.command = UIC_CMD_DME_HIBER_EXIT; 2242 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd); 2243 if (ret) { 2244 ufshcd_set_link_off(hba); 2245 ret = ufshcd_host_reset_and_restore(hba); 2246 } 2247 2248 return ret; 2249} 2250 2251 /** 2252 * ufshcd_init_pwr_info - setting the POR (power on reset) 2253 * values in hba power info 2254 * @hba: per-adapter instance 2255 */ 2256static void ufshcd_init_pwr_info(struct ufs_hba *hba) 2257{ 2258 hba->pwr_info.gear_rx = UFS_PWM_G1; 2259 hba->pwr_info.gear_tx = UFS_PWM_G1; 2260 hba->pwr_info.lane_rx = 1; 2261 hba->pwr_info.lane_tx = 1; 2262 hba->pwr_info.pwr_rx = SLOWAUTO_MODE; 2263 hba->pwr_info.pwr_tx = SLOWAUTO_MODE; 2264 hba->pwr_info.hs_rate = 0; 2265} 2266 2267/** 2268 * ufshcd_get_max_pwr_mode - reads the max power mode negotiated with device 2269 * @hba: per-adapter instance 2270 */ 2271static int ufshcd_get_max_pwr_mode(struct ufs_hba *hba) 2272{ 2273 struct ufs_pa_layer_attr *pwr_info = &hba->max_pwr_info.info; 2274 2275 if (hba->max_pwr_info.is_valid) 2276 return 0; 2277 2278 pwr_info->pwr_tx = FASTAUTO_MODE; 2279 pwr_info->pwr_rx = FASTAUTO_MODE; 2280 pwr_info->hs_rate = PA_HS_MODE_B; 2281 2282 /* Get the connected lane count */ 2283 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDRXDATALANES), 2284 &pwr_info->lane_rx); 2285 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES), 2286 &pwr_info->lane_tx); 2287 2288 if (!pwr_info->lane_rx || !pwr_info->lane_tx) { 2289 dev_err(hba->dev, "%s: invalid connected lanes value. rx=%d, tx=%d\n", 2290 __func__, 2291 pwr_info->lane_rx, 2292 pwr_info->lane_tx); 2293 return -EINVAL; 2294 } 2295 2296 /* 2297 * First, get the maximum gears of HS speed. 2298 * If a zero value, it means there is no HSGEAR capability. 2299 * Then, get the maximum gears of PWM speed. 2300 */ 2301 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR), &pwr_info->gear_rx); 2302 if (!pwr_info->gear_rx) { 2303 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR), 2304 &pwr_info->gear_rx); 2305 if (!pwr_info->gear_rx) { 2306 dev_err(hba->dev, "%s: invalid max pwm rx gear read = %d\n", 2307 __func__, pwr_info->gear_rx); 2308 return -EINVAL; 2309 } 2310 pwr_info->pwr_rx = SLOWAUTO_MODE; 2311 } 2312 2313 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR), 2314 &pwr_info->gear_tx); 2315 if (!pwr_info->gear_tx) { 2316 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR), 2317 &pwr_info->gear_tx); 2318 if (!pwr_info->gear_tx) { 2319 dev_err(hba->dev, "%s: invalid max pwm tx gear read = %d\n", 2320 __func__, pwr_info->gear_tx); 2321 return -EINVAL; 2322 } 2323 pwr_info->pwr_tx = SLOWAUTO_MODE; 2324 } 2325 2326 hba->max_pwr_info.is_valid = true; 2327 return 0; 2328} 2329 2330static int ufshcd_change_power_mode(struct ufs_hba *hba, 2331 struct ufs_pa_layer_attr *pwr_mode) 2332{ 2333 int ret; 2334 2335 /* if already configured to the requested pwr_mode */ 2336 if (pwr_mode->gear_rx == hba->pwr_info.gear_rx && 2337 pwr_mode->gear_tx == hba->pwr_info.gear_tx && 2338 pwr_mode->lane_rx == hba->pwr_info.lane_rx && 2339 pwr_mode->lane_tx == hba->pwr_info.lane_tx && 2340 pwr_mode->pwr_rx == hba->pwr_info.pwr_rx && 2341 pwr_mode->pwr_tx == hba->pwr_info.pwr_tx && 2342 pwr_mode->hs_rate == hba->pwr_info.hs_rate) { 2343 dev_dbg(hba->dev, "%s: power already configured\n", __func__); 2344 return 0; 2345 } 2346 2347 /* 2348 * Configure attributes for power mode change with below. 2349 * - PA_RXGEAR, PA_ACTIVERXDATALANES, PA_RXTERMINATION, 2350 * - PA_TXGEAR, PA_ACTIVETXDATALANES, PA_TXTERMINATION, 2351 * - PA_HSSERIES 2352 */ 2353 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXGEAR), pwr_mode->gear_rx); 2354 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVERXDATALANES), 2355 pwr_mode->lane_rx); 2356 if (pwr_mode->pwr_rx == FASTAUTO_MODE || 2357 pwr_mode->pwr_rx == FAST_MODE) 2358 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), TRUE); 2359 else 2360 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), FALSE); 2361 2362 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXGEAR), pwr_mode->gear_tx); 2363 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVETXDATALANES), 2364 pwr_mode->lane_tx); 2365 if (pwr_mode->pwr_tx == FASTAUTO_MODE || 2366 pwr_mode->pwr_tx == FAST_MODE) 2367 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), TRUE); 2368 else 2369 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), FALSE); 2370 2371 if (pwr_mode->pwr_rx == FASTAUTO_MODE || 2372 pwr_mode->pwr_tx == FASTAUTO_MODE || 2373 pwr_mode->pwr_rx == FAST_MODE || 2374 pwr_mode->pwr_tx == FAST_MODE) 2375 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HSSERIES), 2376 pwr_mode->hs_rate); 2377 2378 ret = ufshcd_uic_change_pwr_mode(hba, pwr_mode->pwr_rx << 4 2379 | pwr_mode->pwr_tx); 2380 2381 if (ret) { 2382 dev_err(hba->dev, 2383 "%s: power mode change failed %d\n", __func__, ret); 2384 } else { 2385 if (hba->vops && hba->vops->pwr_change_notify) 2386 hba->vops->pwr_change_notify(hba, 2387 POST_CHANGE, NULL, pwr_mode); 2388 2389 memcpy(&hba->pwr_info, pwr_mode, 2390 sizeof(struct ufs_pa_layer_attr)); 2391 } 2392 2393 return ret; 2394} 2395 2396/** 2397 * ufshcd_config_pwr_mode - configure a new power mode 2398 * @hba: per-adapter instance 2399 * @desired_pwr_mode: desired power configuration 2400 */ 2401static int ufshcd_config_pwr_mode(struct ufs_hba *hba, 2402 struct ufs_pa_layer_attr *desired_pwr_mode) 2403{ 2404 struct ufs_pa_layer_attr final_params = { 0 }; 2405 int ret; 2406 2407 if (hba->vops && hba->vops->pwr_change_notify) 2408 hba->vops->pwr_change_notify(hba, 2409 PRE_CHANGE, desired_pwr_mode, &final_params); 2410 else 2411 memcpy(&final_params, desired_pwr_mode, sizeof(final_params)); 2412 2413 ret = ufshcd_change_power_mode(hba, &final_params); 2414 2415 return ret; 2416} 2417 2418/** 2419 * ufshcd_complete_dev_init() - checks device readiness 2420 * hba: per-adapter instance 2421 * 2422 * Set fDeviceInit flag and poll until device toggles it. 2423 */ 2424static int ufshcd_complete_dev_init(struct ufs_hba *hba) 2425{ 2426 int i, retries, err = 0; 2427 bool flag_res = 1; 2428 2429 for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) { 2430 /* Set the fDeviceInit flag */ 2431 err = ufshcd_query_flag(hba, UPIU_QUERY_OPCODE_SET_FLAG, 2432 QUERY_FLAG_IDN_FDEVICEINIT, NULL); 2433 if (!err || err == -ETIMEDOUT) 2434 break; 2435 dev_dbg(hba->dev, "%s: error %d retrying\n", __func__, err); 2436 } 2437 if (err) { 2438 dev_err(hba->dev, 2439 "%s setting fDeviceInit flag failed with error %d\n", 2440 __func__, err); 2441 goto out; 2442 } 2443 2444 /* poll for max. 100 iterations for fDeviceInit flag to clear */ 2445 for (i = 0; i < 100 && !err && flag_res; i++) { 2446 for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) { 2447 err = ufshcd_query_flag(hba, 2448 UPIU_QUERY_OPCODE_READ_FLAG, 2449 QUERY_FLAG_IDN_FDEVICEINIT, &flag_res); 2450 if (!err || err == -ETIMEDOUT) 2451 break; 2452 dev_dbg(hba->dev, "%s: error %d retrying\n", __func__, 2453 err); 2454 } 2455 } 2456 if (err) 2457 dev_err(hba->dev, 2458 "%s reading fDeviceInit flag failed with error %d\n", 2459 __func__, err); 2460 else if (flag_res) 2461 dev_err(hba->dev, 2462 "%s fDeviceInit was not cleared by the device\n", 2463 __func__); 2464 2465out: 2466 return err; 2467} 2468 2469/** 2470 * ufshcd_make_hba_operational - Make UFS controller operational 2471 * @hba: per adapter instance 2472 * 2473 * To bring UFS host controller to operational state, 2474 * 1. Enable required interrupts 2475 * 2. Configure interrupt aggregation 2476 * 3. Program UTRL and UTMRL base addres 2477 * 4. Configure run-stop-registers 2478 * 2479 * Returns 0 on success, non-zero value on failure 2480 */ 2481static int ufshcd_make_hba_operational(struct ufs_hba *hba) 2482{ 2483 int err = 0; 2484 u32 reg; 2485 2486 /* Enable required interrupts */ 2487 ufshcd_enable_intr(hba, UFSHCD_ENABLE_INTRS); 2488 2489 /* Configure interrupt aggregation */ 2490 ufshcd_config_intr_aggr(hba, hba->nutrs - 1, INT_AGGR_DEF_TO); 2491 2492 /* Configure UTRL and UTMRL base address registers */ 2493 ufshcd_writel(hba, lower_32_bits(hba->utrdl_dma_addr), 2494 REG_UTP_TRANSFER_REQ_LIST_BASE_L); 2495 ufshcd_writel(hba, upper_32_bits(hba->utrdl_dma_addr), 2496 REG_UTP_TRANSFER_REQ_LIST_BASE_H); 2497 ufshcd_writel(hba, lower_32_bits(hba->utmrdl_dma_addr), 2498 REG_UTP_TASK_REQ_LIST_BASE_L); 2499 ufshcd_writel(hba, upper_32_bits(hba->utmrdl_dma_addr), 2500 REG_UTP_TASK_REQ_LIST_BASE_H); 2501 2502 /* 2503 * UCRDY, UTMRLDY and UTRLRDY bits must be 1 2504 * DEI, HEI bits must be 0 2505 */ 2506 reg = ufshcd_readl(hba, REG_CONTROLLER_STATUS); 2507 if (!(ufshcd_get_lists_status(reg))) { 2508 ufshcd_enable_run_stop_reg(hba); 2509 } else { 2510 dev_err(hba->dev, 2511 "Host controller not ready to process requests"); 2512 err = -EIO; 2513 goto out; 2514 } 2515 2516out: 2517 return err; 2518} 2519 2520/** 2521 * ufshcd_hba_enable - initialize the controller 2522 * @hba: per adapter instance 2523 * 2524 * The controller resets itself and controller firmware initialization 2525 * sequence kicks off. When controller is ready it will set 2526 * the Host Controller Enable bit to 1. 2527 * 2528 * Returns 0 on success, non-zero value on failure 2529 */ 2530static int ufshcd_hba_enable(struct ufs_hba *hba) 2531{ 2532 int retry; 2533 2534 /* 2535 * msleep of 1 and 5 used in this function might result in msleep(20), 2536 * but it was necessary to send the UFS FPGA to reset mode during 2537 * development and testing of this driver. msleep can be changed to 2538 * mdelay and retry count can be reduced based on the controller. 2539 */ 2540 if (!ufshcd_is_hba_active(hba)) { 2541 2542 /* change controller state to "reset state" */ 2543 ufshcd_hba_stop(hba); 2544 2545 /* 2546 * This delay is based on the testing done with UFS host 2547 * controller FPGA. The delay can be changed based on the 2548 * host controller used. 2549 */ 2550 msleep(5); 2551 } 2552 2553 /* UniPro link is disabled at this point */ 2554 ufshcd_set_link_off(hba); 2555 2556 if (hba->vops && hba->vops->hce_enable_notify) 2557 hba->vops->hce_enable_notify(hba, PRE_CHANGE); 2558 2559 /* start controller initialization sequence */ 2560 ufshcd_hba_start(hba); 2561 2562 /* 2563 * To initialize a UFS host controller HCE bit must be set to 1. 2564 * During initialization the HCE bit value changes from 1->0->1. 2565 * When the host controller completes initialization sequence 2566 * it sets the value of HCE bit to 1. The same HCE bit is read back 2567 * to check if the controller has completed initialization sequence. 2568 * So without this delay the value HCE = 1, set in the previous 2569 * instruction might be read back. 2570 * This delay can be changed based on the controller. 2571 */ 2572 msleep(1); 2573 2574 /* wait for the host controller to complete initialization */ 2575 retry = 10; 2576 while (ufshcd_is_hba_active(hba)) { 2577 if (retry) { 2578 retry--; 2579 } else { 2580 dev_err(hba->dev, 2581 "Controller enable failed\n"); 2582 return -EIO; 2583 } 2584 msleep(5); 2585 } 2586 2587 /* enable UIC related interrupts */ 2588 ufshcd_enable_intr(hba, UFSHCD_UIC_MASK); 2589 2590 if (hba->vops && hba->vops->hce_enable_notify) 2591 hba->vops->hce_enable_notify(hba, POST_CHANGE); 2592 2593 return 0; 2594} 2595 2596/** 2597 * ufshcd_link_startup - Initialize unipro link startup 2598 * @hba: per adapter instance 2599 * 2600 * Returns 0 for success, non-zero in case of failure 2601 */ 2602static int ufshcd_link_startup(struct ufs_hba *hba) 2603{ 2604 int ret; 2605 int retries = DME_LINKSTARTUP_RETRIES; 2606 2607 do { 2608 if (hba->vops && hba->vops->link_startup_notify) 2609 hba->vops->link_startup_notify(hba, PRE_CHANGE); 2610 2611 ret = ufshcd_dme_link_startup(hba); 2612 2613 /* check if device is detected by inter-connect layer */ 2614 if (!ret && !ufshcd_is_device_present(hba)) { 2615 dev_err(hba->dev, "%s: Device not present\n", __func__); 2616 ret = -ENXIO; 2617 goto out; 2618 } 2619 2620 /* 2621 * DME link lost indication is only received when link is up, 2622 * but we can't be sure if the link is up until link startup 2623 * succeeds. So reset the local Uni-Pro and try again. 2624 */ 2625 if (ret && ufshcd_hba_enable(hba)) 2626 goto out; 2627 } while (ret && retries--); 2628 2629 if (ret) 2630 /* failed to get the link up... retire */ 2631 goto out; 2632 2633 /* Include any host controller configuration via UIC commands */ 2634 if (hba->vops && hba->vops->link_startup_notify) { 2635 ret = hba->vops->link_startup_notify(hba, POST_CHANGE); 2636 if (ret) 2637 goto out; 2638 } 2639 2640 ret = ufshcd_make_hba_operational(hba); 2641out: 2642 if (ret) 2643 dev_err(hba->dev, "link startup failed %d\n", ret); 2644 return ret; 2645} 2646 2647/** 2648 * ufshcd_verify_dev_init() - Verify device initialization 2649 * @hba: per-adapter instance 2650 * 2651 * Send NOP OUT UPIU and wait for NOP IN response to check whether the 2652 * device Transport Protocol (UTP) layer is ready after a reset. 2653 * If the UTP layer at the device side is not initialized, it may 2654 * not respond with NOP IN UPIU within timeout of %NOP_OUT_TIMEOUT 2655 * and we retry sending NOP OUT for %NOP_OUT_RETRIES iterations. 2656 */ 2657static int ufshcd_verify_dev_init(struct ufs_hba *hba) 2658{ 2659 int err = 0; 2660 int retries; 2661 2662 ufshcd_hold(hba, false); 2663 mutex_lock(&hba->dev_cmd.lock); 2664 for (retries = NOP_OUT_RETRIES; retries > 0; retries--) { 2665 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_NOP, 2666 NOP_OUT_TIMEOUT); 2667 2668 if (!err || err == -ETIMEDOUT) 2669 break; 2670 2671 dev_dbg(hba->dev, "%s: error %d retrying\n", __func__, err); 2672 } 2673 mutex_unlock(&hba->dev_cmd.lock); 2674 ufshcd_release(hba); 2675 2676 if (err) 2677 dev_err(hba->dev, "%s: NOP OUT failed %d\n", __func__, err); 2678 return err; 2679} 2680 2681/** 2682 * ufshcd_set_queue_depth - set lun queue depth 2683 * @sdev: pointer to SCSI device 2684 * 2685 * Read bLUQueueDepth value and activate scsi tagged command 2686 * queueing. For WLUN, queue depth is set to 1. For best-effort 2687 * cases (bLUQueueDepth = 0) the queue depth is set to a maximum 2688 * value that host can queue. 2689 */ 2690static void ufshcd_set_queue_depth(struct scsi_device *sdev) 2691{ 2692 int ret = 0; 2693 u8 lun_qdepth; 2694 struct ufs_hba *hba; 2695 2696 hba = shost_priv(sdev->host); 2697 2698 lun_qdepth = hba->nutrs; 2699 ret = ufshcd_read_unit_desc_param(hba, 2700 ufshcd_scsi_to_upiu_lun(sdev->lun), 2701 UNIT_DESC_PARAM_LU_Q_DEPTH, 2702 &lun_qdepth, 2703 sizeof(lun_qdepth)); 2704 2705 /* Some WLUN doesn't support unit descriptor */ 2706 if (ret == -EOPNOTSUPP) 2707 lun_qdepth = 1; 2708 else if (!lun_qdepth) 2709 /* eventually, we can figure out the real queue depth */ 2710 lun_qdepth = hba->nutrs; 2711 else 2712 lun_qdepth = min_t(int, lun_qdepth, hba->nutrs); 2713 2714 dev_dbg(hba->dev, "%s: activate tcq with queue depth %d\n", 2715 __func__, lun_qdepth); 2716 scsi_activate_tcq(sdev, lun_qdepth); 2717} 2718 2719/* 2720 * ufshcd_get_lu_wp - returns the "b_lu_write_protect" from UNIT DESCRIPTOR 2721 * @hba: per-adapter instance 2722 * @lun: UFS device lun id 2723 * @b_lu_write_protect: pointer to buffer to hold the LU's write protect info 2724 * 2725 * Returns 0 in case of success and b_lu_write_protect status would be returned 2726 * @b_lu_write_protect parameter. 2727 * Returns -ENOTSUPP if reading b_lu_write_protect is not supported. 2728 * Returns -EINVAL in case of invalid parameters passed to this function. 2729 */ 2730static int ufshcd_get_lu_wp(struct ufs_hba *hba, 2731 u8 lun, 2732 u8 *b_lu_write_protect) 2733{ 2734 int ret; 2735 2736 if (!b_lu_write_protect) 2737 ret = -EINVAL; 2738 /* 2739 * According to UFS device spec, RPMB LU can't be write 2740 * protected so skip reading bLUWriteProtect parameter for 2741 * it. For other W-LUs, UNIT DESCRIPTOR is not available. 2742 */ 2743 else if (lun >= UFS_UPIU_MAX_GENERAL_LUN) 2744 ret = -ENOTSUPP; 2745 else 2746 ret = ufshcd_read_unit_desc_param(hba, 2747 lun, 2748 UNIT_DESC_PARAM_LU_WR_PROTECT, 2749 b_lu_write_protect, 2750 sizeof(*b_lu_write_protect)); 2751 return ret; 2752} 2753 2754/** 2755 * ufshcd_get_lu_power_on_wp_status - get LU's power on write protect 2756 * status 2757 * @hba: per-adapter instance 2758 * @sdev: pointer to SCSI device 2759 * 2760 */ 2761static inline void ufshcd_get_lu_power_on_wp_status(struct ufs_hba *hba, 2762 struct scsi_device *sdev) 2763{ 2764 if (hba->dev_info.f_power_on_wp_en && 2765 !hba->dev_info.is_lu_power_on_wp) { 2766 u8 b_lu_write_protect; 2767 2768 if (!ufshcd_get_lu_wp(hba, ufshcd_scsi_to_upiu_lun(sdev->lun), 2769 &b_lu_write_protect) && 2770 (b_lu_write_protect == UFS_LU_POWER_ON_WP)) 2771 hba->dev_info.is_lu_power_on_wp = true; 2772 } 2773} 2774 2775/** 2776 * ufshcd_slave_alloc - handle initial SCSI device configurations 2777 * @sdev: pointer to SCSI device 2778 * 2779 * Returns success 2780 */ 2781static int ufshcd_slave_alloc(struct scsi_device *sdev) 2782{ 2783 struct ufs_hba *hba; 2784 2785 hba = shost_priv(sdev->host); 2786 sdev->tagged_supported = 1; 2787 2788 /* Mode sense(6) is not supported by UFS, so use Mode sense(10) */ 2789 sdev->use_10_for_ms = 1; 2790 scsi_set_tag_type(sdev, MSG_SIMPLE_TAG); 2791 2792 /* allow SCSI layer to restart the device in case of errors */ 2793 sdev->allow_restart = 1; 2794 2795 /* REPORT SUPPORTED OPERATION CODES is not supported */ 2796 sdev->no_report_opcodes = 1; 2797 2798 2799 ufshcd_set_queue_depth(sdev); 2800 2801 ufshcd_get_lu_power_on_wp_status(hba, sdev); 2802 2803 return 0; 2804} 2805 2806/** 2807 * ufshcd_change_queue_depth - change queue depth 2808 * @sdev: pointer to SCSI device 2809 * @depth: required depth to set 2810 * @reason: reason for changing the depth 2811 * 2812 * Change queue depth according to the reason and make sure 2813 * the max. limits are not crossed. 2814 */ 2815static int ufshcd_change_queue_depth(struct scsi_device *sdev, 2816 int depth, int reason) 2817{ 2818 struct ufs_hba *hba = shost_priv(sdev->host); 2819 2820 if (depth > hba->nutrs) 2821 depth = hba->nutrs; 2822 2823 switch (reason) { 2824 case SCSI_QDEPTH_DEFAULT: 2825 case SCSI_QDEPTH_RAMP_UP: 2826 if (!sdev->tagged_supported) 2827 depth = 1; 2828 scsi_adjust_queue_depth(sdev, scsi_get_tag_type(sdev), depth); 2829 break; 2830 case SCSI_QDEPTH_QFULL: 2831 scsi_track_queue_full(sdev, depth); 2832 break; 2833 default: 2834 return -EOPNOTSUPP; 2835 } 2836 2837 return depth; 2838} 2839 2840/** 2841 * ufshcd_slave_configure - adjust SCSI device configurations 2842 * @sdev: pointer to SCSI device 2843 */ 2844static int ufshcd_slave_configure(struct scsi_device *sdev) 2845{ 2846 struct request_queue *q = sdev->request_queue; 2847 2848 blk_queue_update_dma_pad(q, PRDT_DATA_BYTE_COUNT_PAD - 1); 2849 blk_queue_max_segment_size(q, PRDT_DATA_BYTE_COUNT_MAX); 2850 2851 return 0; 2852} 2853 2854/** 2855 * ufshcd_slave_destroy - remove SCSI device configurations 2856 * @sdev: pointer to SCSI device 2857 */ 2858static void ufshcd_slave_destroy(struct scsi_device *sdev) 2859{ 2860 struct ufs_hba *hba; 2861 2862 hba = shost_priv(sdev->host); 2863 scsi_deactivate_tcq(sdev, hba->nutrs); 2864 /* Drop the reference as it won't be needed anymore */ 2865 if (ufshcd_scsi_to_upiu_lun(sdev->lun) == UFS_UPIU_UFS_DEVICE_WLUN) { 2866 unsigned long flags; 2867 2868 spin_lock_irqsave(hba->host->host_lock, flags); 2869 hba->sdev_ufs_device = NULL; 2870 spin_unlock_irqrestore(hba->host->host_lock, flags); 2871 } 2872} 2873 2874/** 2875 * ufshcd_task_req_compl - handle task management request completion 2876 * @hba: per adapter instance 2877 * @index: index of the completed request 2878 * @resp: task management service response 2879 * 2880 * Returns non-zero value on error, zero on success 2881 */ 2882static int ufshcd_task_req_compl(struct ufs_hba *hba, u32 index, u8 *resp) 2883{ 2884 struct utp_task_req_desc *task_req_descp; 2885 struct utp_upiu_task_rsp *task_rsp_upiup; 2886 unsigned long flags; 2887 int ocs_value; 2888 int task_result; 2889 2890 spin_lock_irqsave(hba->host->host_lock, flags); 2891 2892 /* Clear completed tasks from outstanding_tasks */ 2893 __clear_bit(index, &hba->outstanding_tasks); 2894 2895 task_req_descp = hba->utmrdl_base_addr; 2896 ocs_value = ufshcd_get_tmr_ocs(&task_req_descp[index]); 2897 2898 if (ocs_value == OCS_SUCCESS) { 2899 task_rsp_upiup = (struct utp_upiu_task_rsp *) 2900 task_req_descp[index].task_rsp_upiu; 2901 task_result = be32_to_cpu(task_rsp_upiup->header.dword_1); 2902 task_result = ((task_result & MASK_TASK_RESPONSE) >> 8); 2903 if (resp) 2904 *resp = (u8)task_result; 2905 } else { 2906 dev_err(hba->dev, "%s: failed, ocs = 0x%x\n", 2907 __func__, ocs_value); 2908 } 2909 spin_unlock_irqrestore(hba->host->host_lock, flags); 2910 2911 return ocs_value; 2912} 2913 2914/** 2915 * ufshcd_scsi_cmd_status - Update SCSI command result based on SCSI status 2916 * @lrb: pointer to local reference block of completed command 2917 * @scsi_status: SCSI command status 2918 * 2919 * Returns value base on SCSI command status 2920 */ 2921static inline int 2922ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status) 2923{ 2924 int result = 0; 2925 2926 switch (scsi_status) { 2927 case SAM_STAT_CHECK_CONDITION: 2928 ufshcd_copy_sense_data(lrbp); 2929 case SAM_STAT_GOOD: 2930 result |= DID_OK << 16 | 2931 COMMAND_COMPLETE << 8 | 2932 scsi_status; 2933 break; 2934 case SAM_STAT_TASK_SET_FULL: 2935 case SAM_STAT_BUSY: 2936 case SAM_STAT_TASK_ABORTED: 2937 ufshcd_copy_sense_data(lrbp); 2938 result |= scsi_status; 2939 break; 2940 default: 2941 result |= DID_ERROR << 16; 2942 break; 2943 } /* end of switch */ 2944 2945 return result; 2946} 2947 2948/** 2949 * ufshcd_transfer_rsp_status - Get overall status of the response 2950 * @hba: per adapter instance 2951 * @lrb: pointer to local reference block of completed command 2952 * 2953 * Returns result of the command to notify SCSI midlayer 2954 */ 2955static inline int 2956ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) 2957{ 2958 int result = 0; 2959 int scsi_status; 2960 int ocs; 2961 2962 /* overall command status of utrd */ 2963 ocs = ufshcd_get_tr_ocs(lrbp); 2964 2965 switch (ocs) { 2966 case OCS_SUCCESS: 2967 result = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr); 2968 2969 switch (result) { 2970 case UPIU_TRANSACTION_RESPONSE: 2971 /* 2972 * get the response UPIU result to extract 2973 * the SCSI command status 2974 */ 2975 result = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr); 2976 2977 /* 2978 * get the result based on SCSI status response 2979 * to notify the SCSI midlayer of the command status 2980 */ 2981 scsi_status = result & MASK_SCSI_STATUS; 2982 result = ufshcd_scsi_cmd_status(lrbp, scsi_status); 2983 2984 if (ufshcd_is_exception_event(lrbp->ucd_rsp_ptr)) 2985 schedule_work(&hba->eeh_work); 2986 break; 2987 case UPIU_TRANSACTION_REJECT_UPIU: 2988 /* TODO: handle Reject UPIU Response */ 2989 result = DID_ERROR << 16; 2990 dev_err(hba->dev, 2991 "Reject UPIU not fully implemented\n"); 2992 break; 2993 default: 2994 result = DID_ERROR << 16; 2995 dev_err(hba->dev, 2996 "Unexpected request response code = %x\n", 2997 result); 2998 break; 2999 } 3000 break; 3001 case OCS_ABORTED: 3002 result |= DID_ABORT << 16; 3003 break; 3004 case OCS_INVALID_COMMAND_STATUS: 3005 result |= DID_REQUEUE << 16; 3006 break; 3007 case OCS_INVALID_CMD_TABLE_ATTR: 3008 case OCS_INVALID_PRDT_ATTR: 3009 case OCS_MISMATCH_DATA_BUF_SIZE: 3010 case OCS_MISMATCH_RESP_UPIU_SIZE: 3011 case OCS_PEER_COMM_FAILURE: 3012 case OCS_FATAL_ERROR: 3013 default: 3014 result |= DID_ERROR << 16; 3015 dev_err(hba->dev, 3016 "OCS error from controller = %x\n", ocs); 3017 break; 3018 } /* end of switch */ 3019 3020 return result; 3021} 3022 3023/** 3024 * ufshcd_uic_cmd_compl - handle completion of uic command 3025 * @hba: per adapter instance 3026 * @intr_status: interrupt status generated by the controller 3027 */ 3028static void ufshcd_uic_cmd_compl(struct ufs_hba *hba, u32 intr_status) 3029{ 3030 if ((intr_status & UIC_COMMAND_COMPL) && hba->active_uic_cmd) { 3031 hba->active_uic_cmd->argument2 |= 3032 ufshcd_get_uic_cmd_result(hba); 3033 hba->active_uic_cmd->argument3 = 3034 ufshcd_get_dme_attr_val(hba); 3035 complete(&hba->active_uic_cmd->done); 3036 } 3037 3038 if ((intr_status & UFSHCD_UIC_PWR_MASK) && hba->uic_async_done) 3039 complete(hba->uic_async_done); 3040} 3041 3042/** 3043 * ufshcd_transfer_req_compl - handle SCSI and query command completion 3044 * @hba: per adapter instance 3045 */ 3046static void ufshcd_transfer_req_compl(struct ufs_hba *hba) 3047{ 3048 struct ufshcd_lrb *lrbp; 3049 struct scsi_cmnd *cmd; 3050 unsigned long completed_reqs; 3051 u32 tr_doorbell; 3052 int result; 3053 int index; 3054 3055 /* Resetting interrupt aggregation counters first and reading the 3056 * DOOR_BELL afterward allows us to handle all the completed requests. 3057 * In order to prevent other interrupts starvation the DB is read once 3058 * after reset. The down side of this solution is the possibility of 3059 * false interrupt if device completes another request after resetting 3060 * aggregation and before reading the DB. 3061 */ 3062 ufshcd_reset_intr_aggr(hba); 3063 3064 tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL); 3065 completed_reqs = tr_doorbell ^ hba->outstanding_reqs; 3066 3067 for_each_set_bit(index, &completed_reqs, hba->nutrs) { 3068 lrbp = &hba->lrb[index]; 3069 cmd = lrbp->cmd; 3070 if (cmd) { 3071 result = ufshcd_transfer_rsp_status(hba, lrbp); 3072 scsi_dma_unmap(cmd); 3073 cmd->result = result; 3074 /* Mark completed command as NULL in LRB */ 3075 lrbp->cmd = NULL; 3076 clear_bit_unlock(index, &hba->lrb_in_use); 3077 /* Do not touch lrbp after scsi done */ 3078 cmd->scsi_done(cmd); 3079 __ufshcd_release(hba); 3080 } else if (lrbp->command_type == UTP_CMD_TYPE_DEV_MANAGE) { 3081 if (hba->dev_cmd.complete) 3082 complete(hba->dev_cmd.complete); 3083 } 3084 } 3085 3086 /* clear corresponding bits of completed commands */ 3087 hba->outstanding_reqs ^= completed_reqs; 3088 3089 ufshcd_clk_scaling_update_busy(hba); 3090 3091 /* we might have free'd some tags above */ 3092 wake_up(&hba->dev_cmd.tag_wq); 3093} 3094 3095/** 3096 * ufshcd_disable_ee - disable exception event 3097 * @hba: per-adapter instance 3098 * @mask: exception event to disable 3099 * 3100 * Disables exception event in the device so that the EVENT_ALERT 3101 * bit is not set. 3102 * 3103 * Returns zero on success, non-zero error value on failure. 3104 */ 3105static int ufshcd_disable_ee(struct ufs_hba *hba, u16 mask) 3106{ 3107 int err = 0; 3108 u32 val; 3109 3110 if (!(hba->ee_ctrl_mask & mask)) 3111 goto out; 3112 3113 val = hba->ee_ctrl_mask & ~mask; 3114 val &= 0xFFFF; /* 2 bytes */ 3115 err = ufshcd_query_attr(hba, UPIU_QUERY_OPCODE_WRITE_ATTR, 3116 QUERY_ATTR_IDN_EE_CONTROL, 0, 0, &val); 3117 if (!err) 3118 hba->ee_ctrl_mask &= ~mask; 3119out: 3120 return err; 3121} 3122 3123/** 3124 * ufshcd_enable_ee - enable exception event 3125 * @hba: per-adapter instance 3126 * @mask: exception event to enable 3127 * 3128 * Enable corresponding exception event in the device to allow 3129 * device to alert host in critical scenarios. 3130 * 3131 * Returns zero on success, non-zero error value on failure. 3132 */ 3133static int ufshcd_enable_ee(struct ufs_hba *hba, u16 mask) 3134{ 3135 int err = 0; 3136 u32 val; 3137 3138 if (hba->ee_ctrl_mask & mask) 3139 goto out; 3140 3141 val = hba->ee_ctrl_mask | mask; 3142 val &= 0xFFFF; /* 2 bytes */ 3143 err = ufshcd_query_attr(hba, UPIU_QUERY_OPCODE_WRITE_ATTR, 3144 QUERY_ATTR_IDN_EE_CONTROL, 0, 0, &val); 3145 if (!err) 3146 hba->ee_ctrl_mask |= mask; 3147out: 3148 return err; 3149} 3150 3151/** 3152 * ufshcd_enable_auto_bkops - Allow device managed BKOPS 3153 * @hba: per-adapter instance 3154 * 3155 * Allow device to manage background operations on its own. Enabling 3156 * this might lead to inconsistent latencies during normal data transfers 3157 * as the device is allowed to manage its own way of handling background 3158 * operations. 3159 * 3160 * Returns zero on success, non-zero on failure. 3161 */ 3162static int ufshcd_enable_auto_bkops(struct ufs_hba *hba) 3163{ 3164 int err = 0; 3165 3166 if (hba->auto_bkops_enabled) 3167 goto out; 3168 3169 err = ufshcd_query_flag(hba, UPIU_QUERY_OPCODE_SET_FLAG, 3170 QUERY_FLAG_IDN_BKOPS_EN, NULL); 3171 if (err) { 3172 dev_err(hba->dev, "%s: failed to enable bkops %d\n", 3173 __func__, err); 3174 goto out; 3175 } 3176 3177 hba->auto_bkops_enabled = true; 3178 3179 /* No need of URGENT_BKOPS exception from the device */ 3180 err = ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS); 3181 if (err) 3182 dev_err(hba->dev, "%s: failed to disable exception event %d\n", 3183 __func__, err); 3184out: 3185 return err; 3186} 3187 3188/** 3189 * ufshcd_disable_auto_bkops - block device in doing background operations 3190 * @hba: per-adapter instance 3191 * 3192 * Disabling background operations improves command response latency but 3193 * has drawback of device moving into critical state where the device is 3194 * not-operable. Make sure to call ufshcd_enable_auto_bkops() whenever the 3195 * host is idle so that BKOPS are managed effectively without any negative 3196 * impacts. 3197 * 3198 * Returns zero on success, non-zero on failure. 3199 */ 3200static int ufshcd_disable_auto_bkops(struct ufs_hba *hba) 3201{ 3202 int err = 0; 3203 3204 if (!hba->auto_bkops_enabled) 3205 goto out; 3206 3207 /* 3208 * If host assisted BKOPs is to be enabled, make sure 3209 * urgent bkops exception is allowed. 3210 */ 3211 err = ufshcd_enable_ee(hba, MASK_EE_URGENT_BKOPS); 3212 if (err) { 3213 dev_err(hba->dev, "%s: failed to enable exception event %d\n", 3214 __func__, err); 3215 goto out; 3216 } 3217 3218 err = ufshcd_query_flag(hba, UPIU_QUERY_OPCODE_CLEAR_FLAG, 3219 QUERY_FLAG_IDN_BKOPS_EN, NULL); 3220 if (err) { 3221 dev_err(hba->dev, "%s: failed to disable bkops %d\n", 3222 __func__, err); 3223 ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS); 3224 goto out; 3225 } 3226 3227 hba->auto_bkops_enabled = false; 3228out: 3229 return err; 3230} 3231 3232/** 3233 * ufshcd_force_reset_auto_bkops - force enable of auto bkops 3234 * @hba: per adapter instance 3235 * 3236 * After a device reset the device may toggle the BKOPS_EN flag 3237 * to default value. The s/w tracking variables should be updated 3238 * as well. Do this by forcing enable of auto bkops. 3239 */ 3240static void ufshcd_force_reset_auto_bkops(struct ufs_hba *hba) 3241{ 3242 hba->auto_bkops_enabled = false; 3243 hba->ee_ctrl_mask |= MASK_EE_URGENT_BKOPS; 3244 ufshcd_enable_auto_bkops(hba); 3245} 3246 3247static inline int ufshcd_get_bkops_status(struct ufs_hba *hba, u32 *status) 3248{ 3249 return ufshcd_query_attr(hba, UPIU_QUERY_OPCODE_READ_ATTR, 3250 QUERY_ATTR_IDN_BKOPS_STATUS, 0, 0, status); 3251} 3252 3253/** 3254 * ufshcd_bkops_ctrl - control the auto bkops based on current bkops status 3255 * @hba: per-adapter instance 3256 * @status: bkops_status value 3257 * 3258 * Read the bkops_status from the UFS device and Enable fBackgroundOpsEn 3259 * flag in the device to permit background operations if the device 3260 * bkops_status is greater than or equal to "status" argument passed to 3261 * this function, disable otherwise. 3262 * 3263 * Returns 0 for success, non-zero in case of failure. 3264 * 3265 * NOTE: Caller of this function can check the "hba->auto_bkops_enabled" flag 3266 * to know whether auto bkops is enabled or disabled after this function 3267 * returns control to it. 3268 */ 3269static int ufshcd_bkops_ctrl(struct ufs_hba *hba, 3270 enum bkops_status status) 3271{ 3272 int err; 3273 u32 curr_status = 0; 3274 3275 err = ufshcd_get_bkops_status(hba, &curr_status); 3276 if (err) { 3277 dev_err(hba->dev, "%s: failed to get BKOPS status %d\n", 3278 __func__, err); 3279 goto out; 3280 } else if (curr_status > BKOPS_STATUS_MAX) { 3281 dev_err(hba->dev, "%s: invalid BKOPS status %d\n", 3282 __func__, curr_status); 3283 err = -EINVAL; 3284 goto out; 3285 } 3286 3287 if (curr_status >= status) 3288 err = ufshcd_enable_auto_bkops(hba); 3289 else 3290 err = ufshcd_disable_auto_bkops(hba); 3291out: 3292 return err; 3293} 3294 3295/** 3296 * ufshcd_urgent_bkops - handle urgent bkops exception event 3297 * @hba: per-adapter instance 3298 * 3299 * Enable fBackgroundOpsEn flag in the device to permit background 3300 * operations. 3301 * 3302 * If BKOPs is enabled, this function returns 0, 1 if the bkops in not enabled 3303 * and negative error value for any other failure. 3304 */ 3305static int ufshcd_urgent_bkops(struct ufs_hba *hba) 3306{ 3307 return ufshcd_bkops_ctrl(hba, BKOPS_STATUS_PERF_IMPACT); 3308} 3309 3310static inline int ufshcd_get_ee_status(struct ufs_hba *hba, u32 *status) 3311{ 3312 return ufshcd_query_attr(hba, UPIU_QUERY_OPCODE_READ_ATTR, 3313 QUERY_ATTR_IDN_EE_STATUS, 0, 0, status); 3314} 3315 3316/** 3317 * ufshcd_exception_event_handler - handle exceptions raised by device 3318 * @work: pointer to work data 3319 * 3320 * Read bExceptionEventStatus attribute from the device and handle the 3321 * exception event accordingly. 3322 */ 3323static void ufshcd_exception_event_handler(struct work_struct *work) 3324{ 3325 struct ufs_hba *hba; 3326 int err; 3327 u32 status = 0; 3328 hba = container_of(work, struct ufs_hba, eeh_work); 3329 3330 pm_runtime_get_sync(hba->dev); 3331 err = ufshcd_get_ee_status(hba, &status); 3332 if (err) { 3333 dev_err(hba->dev, "%s: failed to get exception status %d\n", 3334 __func__, err); 3335 goto out; 3336 } 3337 3338 status &= hba->ee_ctrl_mask; 3339 if (status & MASK_EE_URGENT_BKOPS) { 3340 err = ufshcd_urgent_bkops(hba); 3341 if (err < 0) 3342 dev_err(hba->dev, "%s: failed to handle urgent bkops %d\n", 3343 __func__, err); 3344 } 3345out: 3346 pm_runtime_put_sync(hba->dev); 3347 return; 3348} 3349 3350/** 3351 * ufshcd_err_handler - handle UFS errors that require s/w attention 3352 * @work: pointer to work structure 3353 */ 3354static void ufshcd_err_handler(struct work_struct *work) 3355{ 3356 struct ufs_hba *hba; 3357 unsigned long flags; 3358 u32 err_xfer = 0; 3359 u32 err_tm = 0; 3360 int err = 0; 3361 int tag; 3362 3363 hba = container_of(work, struct ufs_hba, eh_work); 3364 3365 pm_runtime_get_sync(hba->dev); 3366 ufshcd_hold(hba, false); 3367 3368 spin_lock_irqsave(hba->host->host_lock, flags); 3369 if (hba->ufshcd_state == UFSHCD_STATE_RESET) { 3370 spin_unlock_irqrestore(hba->host->host_lock, flags); 3371 goto out; 3372 } 3373 3374 hba->ufshcd_state = UFSHCD_STATE_RESET; 3375 ufshcd_set_eh_in_progress(hba); 3376 3377 /* Complete requests that have door-bell cleared by h/w */ 3378 ufshcd_transfer_req_compl(hba); 3379 ufshcd_tmc_handler(hba); 3380 spin_unlock_irqrestore(hba->host->host_lock, flags); 3381 3382 /* Clear pending transfer requests */ 3383 for_each_set_bit(tag, &hba->outstanding_reqs, hba->nutrs) 3384 if (ufshcd_clear_cmd(hba, tag)) 3385 err_xfer |= 1 << tag; 3386 3387 /* Clear pending task management requests */ 3388 for_each_set_bit(tag, &hba->outstanding_tasks, hba->nutmrs) 3389 if (ufshcd_clear_tm_cmd(hba, tag)) 3390 err_tm |= 1 << tag; 3391 3392 /* Complete the requests that are cleared by s/w */ 3393 spin_lock_irqsave(hba->host->host_lock, flags); 3394 ufshcd_transfer_req_compl(hba); 3395 ufshcd_tmc_handler(hba); 3396 spin_unlock_irqrestore(hba->host->host_lock, flags); 3397 3398 /* Fatal errors need reset */ 3399 if (err_xfer || err_tm || (hba->saved_err & INT_FATAL_ERRORS) || 3400 ((hba->saved_err & UIC_ERROR) && 3401 (hba->saved_uic_err & UFSHCD_UIC_DL_PA_INIT_ERROR))) { 3402 err = ufshcd_reset_and_restore(hba); 3403 if (err) { 3404 dev_err(hba->dev, "%s: reset and restore failed\n", 3405 __func__); 3406 hba->ufshcd_state = UFSHCD_STATE_ERROR; 3407 } 3408 /* 3409 * Inform scsi mid-layer that we did reset and allow to handle 3410 * Unit Attention properly. 3411 */ 3412 scsi_report_bus_reset(hba->host, 0); 3413 hba->saved_err = 0; 3414 hba->saved_uic_err = 0; 3415 } 3416 ufshcd_clear_eh_in_progress(hba); 3417 3418out: 3419 scsi_unblock_requests(hba->host); 3420 ufshcd_release(hba); 3421 pm_runtime_put_sync(hba->dev); 3422} 3423 3424/** 3425 * ufshcd_update_uic_error - check and set fatal UIC error flags. 3426 * @hba: per-adapter instance 3427 */ 3428static void ufshcd_update_uic_error(struct ufs_hba *hba) 3429{ 3430 u32 reg; 3431 3432 /* PA_INIT_ERROR is fatal and needs UIC reset */ 3433 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DATA_LINK_LAYER); 3434 if (reg & UIC_DATA_LINK_LAYER_ERROR_PA_INIT) 3435 hba->uic_error |= UFSHCD_UIC_DL_PA_INIT_ERROR; 3436 3437 /* UIC NL/TL/DME errors needs software retry */ 3438 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_NETWORK_LAYER); 3439 if (reg) 3440 hba->uic_error |= UFSHCD_UIC_NL_ERROR; 3441 3442 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_TRANSPORT_LAYER); 3443 if (reg) 3444 hba->uic_error |= UFSHCD_UIC_TL_ERROR; 3445 3446 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DME); 3447 if (reg) 3448 hba->uic_error |= UFSHCD_UIC_DME_ERROR; 3449 3450 dev_dbg(hba->dev, "%s: UIC error flags = 0x%08x\n", 3451 __func__, hba->uic_error); 3452} 3453 3454/** 3455 * ufshcd_check_errors - Check for errors that need s/w attention 3456 * @hba: per-adapter instance 3457 */ 3458static void ufshcd_check_errors(struct ufs_hba *hba) 3459{ 3460 bool queue_eh_work = false; 3461 3462 if (hba->errors & INT_FATAL_ERRORS) 3463 queue_eh_work = true; 3464 3465 if (hba->errors & UIC_ERROR) { 3466 hba->uic_error = 0; 3467 ufshcd_update_uic_error(hba); 3468 if (hba->uic_error) 3469 queue_eh_work = true; 3470 } 3471 3472 if (queue_eh_work) { 3473 /* handle fatal errors only when link is functional */ 3474 if (hba->ufshcd_state == UFSHCD_STATE_OPERATIONAL) { 3475 /* block commands from scsi mid-layer */ 3476 scsi_block_requests(hba->host); 3477 3478 /* transfer error masks to sticky bits */ 3479 hba->saved_err |= hba->errors; 3480 hba->saved_uic_err |= hba->uic_error; 3481 3482 hba->ufshcd_state = UFSHCD_STATE_ERROR; 3483 schedule_work(&hba->eh_work); 3484 } 3485 } 3486 /* 3487 * if (!queue_eh_work) - 3488 * Other errors are either non-fatal where host recovers 3489 * itself without s/w intervention or errors that will be 3490 * handled by the SCSI core layer. 3491 */ 3492} 3493 3494/** 3495 * ufshcd_tmc_handler - handle task management function completion 3496 * @hba: per adapter instance 3497 */ 3498static void ufshcd_tmc_handler(struct ufs_hba *hba) 3499{ 3500 u32 tm_doorbell; 3501 3502 tm_doorbell = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL); 3503 hba->tm_condition = tm_doorbell ^ hba->outstanding_tasks; 3504 wake_up(&hba->tm_wq); 3505} 3506 3507/** 3508 * ufshcd_sl_intr - Interrupt service routine 3509 * @hba: per adapter instance 3510 * @intr_status: contains interrupts generated by the controller 3511 */ 3512static void ufshcd_sl_intr(struct ufs_hba *hba, u32 intr_status) 3513{ 3514 hba->errors = UFSHCD_ERROR_MASK & intr_status; 3515 if (hba->errors) 3516 ufshcd_check_errors(hba); 3517 3518 if (intr_status & UFSHCD_UIC_MASK) 3519 ufshcd_uic_cmd_compl(hba, intr_status); 3520 3521 if (intr_status & UTP_TASK_REQ_COMPL) 3522 ufshcd_tmc_handler(hba); 3523 3524 if (intr_status & UTP_TRANSFER_REQ_COMPL) 3525 ufshcd_transfer_req_compl(hba); 3526} 3527 3528/** 3529 * ufshcd_intr - Main interrupt service routine 3530 * @irq: irq number 3531 * @__hba: pointer to adapter instance 3532 * 3533 * Returns IRQ_HANDLED - If interrupt is valid 3534 * IRQ_NONE - If invalid interrupt 3535 */ 3536static irqreturn_t ufshcd_intr(int irq, void *__hba) 3537{ 3538 u32 intr_status; 3539 irqreturn_t retval = IRQ_NONE; 3540 struct ufs_hba *hba = __hba; 3541 3542 spin_lock(hba->host->host_lock); 3543 intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS); 3544 3545 if (intr_status) { 3546 ufshcd_writel(hba, intr_status, REG_INTERRUPT_STATUS); 3547 ufshcd_sl_intr(hba, intr_status); 3548 retval = IRQ_HANDLED; 3549 } 3550 spin_unlock(hba->host->host_lock); 3551 return retval; 3552} 3553 3554static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag) 3555{ 3556 int err = 0; 3557 u32 mask = 1 << tag; 3558 unsigned long flags; 3559 3560 if (!test_bit(tag, &hba->outstanding_tasks)) 3561 goto out; 3562 3563 spin_lock_irqsave(hba->host->host_lock, flags); 3564 ufshcd_writel(hba, ~(1 << tag), REG_UTP_TASK_REQ_LIST_CLEAR); 3565 spin_unlock_irqrestore(hba->host->host_lock, flags); 3566 3567 /* poll for max. 1 sec to clear door bell register by h/w */ 3568 err = ufshcd_wait_for_register(hba, 3569 REG_UTP_TASK_REQ_DOOR_BELL, 3570 mask, 0, 1000, 1000); 3571out: 3572 return err; 3573} 3574 3575/** 3576 * ufshcd_issue_tm_cmd - issues task management commands to controller 3577 * @hba: per adapter instance 3578 * @lun_id: LUN ID to which TM command is sent 3579 * @task_id: task ID to which the TM command is applicable 3580 * @tm_function: task management function opcode 3581 * @tm_response: task management service response return value 3582 * 3583 * Returns non-zero value on error, zero on success. 3584 */ 3585static int ufshcd_issue_tm_cmd(struct ufs_hba *hba, int lun_id, int task_id, 3586 u8 tm_function, u8 *tm_response) 3587{ 3588 struct utp_task_req_desc *task_req_descp; 3589 struct utp_upiu_task_req *task_req_upiup; 3590 struct Scsi_Host *host; 3591 unsigned long flags; 3592 int free_slot; 3593 int err; 3594 int task_tag; 3595 3596 host = hba->host; 3597 3598 /* 3599 * Get free slot, sleep if slots are unavailable. 3600 * Even though we use wait_event() which sleeps indefinitely, 3601 * the maximum wait time is bounded by %TM_CMD_TIMEOUT. 3602 */ 3603 wait_event(hba->tm_tag_wq, ufshcd_get_tm_free_slot(hba, &free_slot)); 3604 ufshcd_hold(hba, false); 3605 3606 spin_lock_irqsave(host->host_lock, flags); 3607 task_req_descp = hba->utmrdl_base_addr; 3608 task_req_descp += free_slot; 3609 3610 /* Configure task request descriptor */ 3611 task_req_descp->header.dword_0 = cpu_to_le32(UTP_REQ_DESC_INT_CMD); 3612 task_req_descp->header.dword_2 = 3613 cpu_to_le32(OCS_INVALID_COMMAND_STATUS); 3614 3615 /* Configure task request UPIU */ 3616 task_req_upiup = 3617 (struct utp_upiu_task_req *) task_req_descp->task_req_upiu; 3618 task_tag = hba->nutrs + free_slot; 3619 task_req_upiup->header.dword_0 = 3620 UPIU_HEADER_DWORD(UPIU_TRANSACTION_TASK_REQ, 0, 3621 lun_id, task_tag); 3622 task_req_upiup->header.dword_1 = 3623 UPIU_HEADER_DWORD(0, tm_function, 0, 0); 3624 /* 3625 * The host shall provide the same value for LUN field in the basic 3626 * header and for Input Parameter. 3627 */ 3628 task_req_upiup->input_param1 = cpu_to_be32(lun_id); 3629 task_req_upiup->input_param2 = cpu_to_be32(task_id); 3630 3631 /* send command to the controller */ 3632 __set_bit(free_slot, &hba->outstanding_tasks); 3633 ufshcd_writel(hba, 1 << free_slot, REG_UTP_TASK_REQ_DOOR_BELL); 3634 3635 spin_unlock_irqrestore(host->host_lock, flags); 3636 3637 /* wait until the task management command is completed */ 3638 err = wait_event_timeout(hba->tm_wq, 3639 test_bit(free_slot, &hba->tm_condition), 3640 msecs_to_jiffies(TM_CMD_TIMEOUT)); 3641 if (!err) { 3642 dev_err(hba->dev, "%s: task management cmd 0x%.2x timed-out\n", 3643 __func__, tm_function); 3644 if (ufshcd_clear_tm_cmd(hba, free_slot)) 3645 dev_WARN(hba->dev, "%s: unable clear tm cmd (slot %d) after timeout\n", 3646 __func__, free_slot); 3647 err = -ETIMEDOUT; 3648 } else { 3649 err = ufshcd_task_req_compl(hba, free_slot, tm_response); 3650 } 3651 3652 clear_bit(free_slot, &hba->tm_condition); 3653 ufshcd_put_tm_slot(hba, free_slot); 3654 wake_up(&hba->tm_tag_wq); 3655 3656 ufshcd_release(hba); 3657 return err; 3658} 3659 3660/** 3661 * ufshcd_eh_device_reset_handler - device reset handler registered to 3662 * scsi layer. 3663 * @cmd: SCSI command pointer 3664 * 3665 * Returns SUCCESS/FAILED 3666 */ 3667static int ufshcd_eh_device_reset_handler(struct scsi_cmnd *cmd) 3668{ 3669 struct Scsi_Host *host; 3670 struct ufs_hba *hba; 3671 unsigned int tag; 3672 u32 pos; 3673 int err; 3674 u8 resp = 0xF; 3675 struct ufshcd_lrb *lrbp; 3676 unsigned long flags; 3677 3678 host = cmd->device->host; 3679 hba = shost_priv(host); 3680 tag = cmd->request->tag; 3681 3682 lrbp = &hba->lrb[tag]; 3683 err = ufshcd_issue_tm_cmd(hba, lrbp->lun, 0, UFS_LOGICAL_RESET, &resp); 3684 if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) { 3685 if (!err) 3686 err = resp; 3687 goto out; 3688 } 3689 3690 /* clear the commands that were pending for corresponding LUN */ 3691 for_each_set_bit(pos, &hba->outstanding_reqs, hba->nutrs) { 3692 if (hba->lrb[pos].lun == lrbp->lun) { 3693 err = ufshcd_clear_cmd(hba, pos); 3694 if (err) 3695 break; 3696 } 3697 } 3698 spin_lock_irqsave(host->host_lock, flags); 3699 ufshcd_transfer_req_compl(hba); 3700 spin_unlock_irqrestore(host->host_lock, flags); 3701out: 3702 if (!err) { 3703 err = SUCCESS; 3704 } else { 3705 dev_err(hba->dev, "%s: failed with err %d\n", __func__, err); 3706 err = FAILED; 3707 } 3708 return err; 3709} 3710 3711/** 3712 * ufshcd_abort - abort a specific command 3713 * @cmd: SCSI command pointer 3714 * 3715 * Abort the pending command in device by sending UFS_ABORT_TASK task management 3716 * command, and in host controller by clearing the door-bell register. There can 3717 * be race between controller sending the command to the device while abort is 3718 * issued. To avoid that, first issue UFS_QUERY_TASK to check if the command is 3719 * really issued and then try to abort it. 3720 * 3721 * Returns SUCCESS/FAILED 3722 */ 3723static int ufshcd_abort(struct scsi_cmnd *cmd) 3724{ 3725 struct Scsi_Host *host; 3726 struct ufs_hba *hba; 3727 unsigned long flags; 3728 unsigned int tag; 3729 int err = 0; 3730 int poll_cnt; 3731 u8 resp = 0xF; 3732 struct ufshcd_lrb *lrbp; 3733 u32 reg; 3734 3735 host = cmd->device->host; 3736 hba = shost_priv(host); 3737 tag = cmd->request->tag; 3738 3739 ufshcd_hold(hba, false); 3740 /* If command is already aborted/completed, return SUCCESS */ 3741 if (!(test_bit(tag, &hba->outstanding_reqs))) 3742 goto out; 3743 3744 reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL); 3745 if (!(reg & (1 << tag))) { 3746 dev_err(hba->dev, 3747 "%s: cmd was completed, but without a notifying intr, tag = %d", 3748 __func__, tag); 3749 } 3750 3751 lrbp = &hba->lrb[tag]; 3752 for (poll_cnt = 100; poll_cnt; poll_cnt--) { 3753 err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag, 3754 UFS_QUERY_TASK, &resp); 3755 if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_SUCCEEDED) { 3756 /* cmd pending in the device */ 3757 break; 3758 } else if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_COMPL) { 3759 /* 3760 * cmd not pending in the device, check if it is 3761 * in transition. 3762 */ 3763 reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL); 3764 if (reg & (1 << tag)) { 3765 /* sleep for max. 200us to stabilize */ 3766 usleep_range(100, 200); 3767 continue; 3768 } 3769 /* command completed already */ 3770 goto out; 3771 } else { 3772 if (!err) 3773 err = resp; /* service response error */ 3774 goto out; 3775 } 3776 } 3777 3778 if (!poll_cnt) { 3779 err = -EBUSY; 3780 goto out; 3781 } 3782 3783 err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag, 3784 UFS_ABORT_TASK, &resp); 3785 if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) { 3786 if (!err) 3787 err = resp; /* service response error */ 3788 goto out; 3789 } 3790 3791 err = ufshcd_clear_cmd(hba, tag); 3792 if (err) 3793 goto out; 3794 3795 scsi_dma_unmap(cmd); 3796 3797 spin_lock_irqsave(host->host_lock, flags); 3798 __clear_bit(tag, &hba->outstanding_reqs); 3799 hba->lrb[tag].cmd = NULL; 3800 spin_unlock_irqrestore(host->host_lock, flags); 3801 3802 clear_bit_unlock(tag, &hba->lrb_in_use); 3803 wake_up(&hba->dev_cmd.tag_wq); 3804 3805out: 3806 if (!err) { 3807 err = SUCCESS; 3808 } else { 3809 dev_err(hba->dev, "%s: failed with err %d\n", __func__, err); 3810 err = FAILED; 3811 } 3812 3813 /* 3814 * This ufshcd_release() corresponds to the original scsi cmd that got 3815 * aborted here (as we won't get any IRQ for it). 3816 */ 3817 ufshcd_release(hba); 3818 return err; 3819} 3820 3821/** 3822 * ufshcd_host_reset_and_restore - reset and restore host controller 3823 * @hba: per-adapter instance 3824 * 3825 * Note that host controller reset may issue DME_RESET to 3826 * local and remote (device) Uni-Pro stack and the attributes 3827 * are reset to default state. 3828 * 3829 * Returns zero on success, non-zero on failure 3830 */ 3831static int ufshcd_host_reset_and_restore(struct ufs_hba *hba) 3832{ 3833 int err; 3834 unsigned long flags; 3835 3836 /* Reset the host controller */ 3837 spin_lock_irqsave(hba->host->host_lock, flags); 3838 ufshcd_hba_stop(hba); 3839 spin_unlock_irqrestore(hba->host->host_lock, flags); 3840 3841 err = ufshcd_hba_enable(hba); 3842 if (err) 3843 goto out; 3844 3845 /* Establish the link again and restore the device */ 3846 err = ufshcd_probe_hba(hba); 3847 3848 if (!err && (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL)) 3849 err = -EIO; 3850out: 3851 if (err) 3852 dev_err(hba->dev, "%s: Host init failed %d\n", __func__, err); 3853 3854 return err; 3855} 3856 3857/** 3858 * ufshcd_reset_and_restore - reset and re-initialize host/device 3859 * @hba: per-adapter instance 3860 * 3861 * Reset and recover device, host and re-establish link. This 3862 * is helpful to recover the communication in fatal error conditions. 3863 * 3864 * Returns zero on success, non-zero on failure 3865 */ 3866static int ufshcd_reset_and_restore(struct ufs_hba *hba) 3867{ 3868 int err = 0; 3869 unsigned long flags; 3870 int retries = MAX_HOST_RESET_RETRIES; 3871 3872 do { 3873 err = ufshcd_host_reset_and_restore(hba); 3874 } while (err && --retries); 3875 3876 /* 3877 * After reset the door-bell might be cleared, complete 3878 * outstanding requests in s/w here. 3879 */ 3880 spin_lock_irqsave(hba->host->host_lock, flags); 3881 ufshcd_transfer_req_compl(hba); 3882 ufshcd_tmc_handler(hba); 3883 spin_unlock_irqrestore(hba->host->host_lock, flags); 3884 3885 return err; 3886} 3887 3888/** 3889 * ufshcd_eh_host_reset_handler - host reset handler registered to scsi layer 3890 * @cmd - SCSI command pointer 3891 * 3892 * Returns SUCCESS/FAILED 3893 */ 3894static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd) 3895{ 3896 int err; 3897 unsigned long flags; 3898 struct ufs_hba *hba; 3899 3900 hba = shost_priv(cmd->device->host); 3901 3902 ufshcd_hold(hba, false); 3903 /* 3904 * Check if there is any race with fatal error handling. 3905 * If so, wait for it to complete. Even though fatal error 3906 * handling does reset and restore in some cases, don't assume 3907 * anything out of it. We are just avoiding race here. 3908 */ 3909 do { 3910 spin_lock_irqsave(hba->host->host_lock, flags); 3911 if (!(work_pending(&hba->eh_work) || 3912 hba->ufshcd_state == UFSHCD_STATE_RESET)) 3913 break; 3914 spin_unlock_irqrestore(hba->host->host_lock, flags); 3915 dev_dbg(hba->dev, "%s: reset in progress\n", __func__); 3916 flush_work(&hba->eh_work); 3917 } while (1); 3918 3919 hba->ufshcd_state = UFSHCD_STATE_RESET; 3920 ufshcd_set_eh_in_progress(hba); 3921 spin_unlock_irqrestore(hba->host->host_lock, flags); 3922 3923 err = ufshcd_reset_and_restore(hba); 3924 3925 spin_lock_irqsave(hba->host->host_lock, flags); 3926 if (!err) { 3927 err = SUCCESS; 3928 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL; 3929 } else { 3930 err = FAILED; 3931 hba->ufshcd_state = UFSHCD_STATE_ERROR; 3932 } 3933 ufshcd_clear_eh_in_progress(hba); 3934 spin_unlock_irqrestore(hba->host->host_lock, flags); 3935 3936 ufshcd_release(hba); 3937 return err; 3938} 3939 3940/** 3941 * ufshcd_get_max_icc_level - calculate the ICC level 3942 * @sup_curr_uA: max. current supported by the regulator 3943 * @start_scan: row at the desc table to start scan from 3944 * @buff: power descriptor buffer 3945 * 3946 * Returns calculated max ICC level for specific regulator 3947 */ 3948static u32 ufshcd_get_max_icc_level(int sup_curr_uA, u32 start_scan, char *buff) 3949{ 3950 int i; 3951 int curr_uA; 3952 u16 data; 3953 u16 unit; 3954 3955 for (i = start_scan; i >= 0; i--) { 3956 data = be16_to_cpu(*((u16 *)(buff + 2*i))); 3957 unit = (data & ATTR_ICC_LVL_UNIT_MASK) >> 3958 ATTR_ICC_LVL_UNIT_OFFSET; 3959 curr_uA = data & ATTR_ICC_LVL_VALUE_MASK; 3960 switch (unit) { 3961 case UFSHCD_NANO_AMP: 3962 curr_uA = curr_uA / 1000; 3963 break; 3964 case UFSHCD_MILI_AMP: 3965 curr_uA = curr_uA * 1000; 3966 break; 3967 case UFSHCD_AMP: 3968 curr_uA = curr_uA * 1000 * 1000; 3969 break; 3970 case UFSHCD_MICRO_AMP: 3971 default: 3972 break; 3973 } 3974 if (sup_curr_uA >= curr_uA) 3975 break; 3976 } 3977 if (i < 0) { 3978 i = 0; 3979 pr_err("%s: Couldn't find valid icc_level = %d", __func__, i); 3980 } 3981 3982 return (u32)i; 3983} 3984 3985/** 3986 * ufshcd_calc_icc_level - calculate the max ICC level 3987 * In case regulators are not initialized we'll return 0 3988 * @hba: per-adapter instance 3989 * @desc_buf: power descriptor buffer to extract ICC levels from. 3990 * @len: length of desc_buff 3991 * 3992 * Returns calculated ICC level 3993 */ 3994static u32 ufshcd_find_max_sup_active_icc_level(struct ufs_hba *hba, 3995 u8 *desc_buf, int len) 3996{ 3997 u32 icc_level = 0; 3998 3999 if (!hba->vreg_info.vcc || !hba->vreg_info.vccq || 4000 !hba->vreg_info.vccq2) { 4001 dev_err(hba->dev, 4002 "%s: Regulator capability was not set, actvIccLevel=%d", 4003 __func__, icc_level); 4004 goto out; 4005 } 4006 4007 if (hba->vreg_info.vcc) 4008 icc_level = ufshcd_get_max_icc_level( 4009 hba->vreg_info.vcc->max_uA, 4010 POWER_DESC_MAX_ACTV_ICC_LVLS - 1, 4011 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCC_0]); 4012 4013 if (hba->vreg_info.vccq) 4014 icc_level = ufshcd_get_max_icc_level( 4015 hba->vreg_info.vccq->max_uA, 4016 icc_level, 4017 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ_0]); 4018 4019 if (hba->vreg_info.vccq2) 4020 icc_level = ufshcd_get_max_icc_level( 4021 hba->vreg_info.vccq2->max_uA, 4022 icc_level, 4023 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ2_0]); 4024out: 4025 return icc_level; 4026} 4027 4028static void ufshcd_init_icc_levels(struct ufs_hba *hba) 4029{ 4030 int ret; 4031 int buff_len = QUERY_DESC_POWER_MAX_SIZE; 4032 u8 desc_buf[QUERY_DESC_POWER_MAX_SIZE]; 4033 4034 ret = ufshcd_read_power_desc(hba, desc_buf, buff_len); 4035 if (ret) { 4036 dev_err(hba->dev, 4037 "%s: Failed reading power descriptor.len = %d ret = %d", 4038 __func__, buff_len, ret); 4039 return; 4040 } 4041 4042 hba->init_prefetch_data.icc_level = 4043 ufshcd_find_max_sup_active_icc_level(hba, 4044 desc_buf, buff_len); 4045 dev_dbg(hba->dev, "%s: setting icc_level 0x%x", 4046 __func__, hba->init_prefetch_data.icc_level); 4047 4048 ret = ufshcd_query_attr(hba, UPIU_QUERY_OPCODE_WRITE_ATTR, 4049 QUERY_ATTR_IDN_ACTIVE_ICC_LVL, 0, 0, 4050 &hba->init_prefetch_data.icc_level); 4051 4052 if (ret) 4053 dev_err(hba->dev, 4054 "%s: Failed configuring bActiveICCLevel = %d ret = %d", 4055 __func__, hba->init_prefetch_data.icc_level , ret); 4056 4057} 4058 4059/** 4060 * ufshcd_scsi_add_wlus - Adds required W-LUs 4061 * @hba: per-adapter instance 4062 * 4063 * UFS device specification requires the UFS devices to support 4 well known 4064 * logical units: 4065 * "REPORT_LUNS" (address: 01h) 4066 * "UFS Device" (address: 50h) 4067 * "RPMB" (address: 44h) 4068 * "BOOT" (address: 30h) 4069 * UFS device's power management needs to be controlled by "POWER CONDITION" 4070 * field of SSU (START STOP UNIT) command. But this "power condition" field 4071 * will take effect only when its sent to "UFS device" well known logical unit 4072 * hence we require the scsi_device instance to represent this logical unit in 4073 * order for the UFS host driver to send the SSU command for power management. 4074 4075 * We also require the scsi_device instance for "RPMB" (Replay Protected Memory 4076 * Block) LU so user space process can control this LU. User space may also 4077 * want to have access to BOOT LU. 4078 4079 * This function adds scsi device instances for each of all well known LUs 4080 * (except "REPORT LUNS" LU). 4081 * 4082 * Returns zero on success (all required W-LUs are added successfully), 4083 * non-zero error value on failure (if failed to add any of the required W-LU). 4084 */ 4085static int ufshcd_scsi_add_wlus(struct ufs_hba *hba) 4086{ 4087 int ret = 0; 4088 struct scsi_device *sdev_rpmb; 4089 struct scsi_device *sdev_boot; 4090 4091 hba->sdev_ufs_device = __scsi_add_device(hba->host, 0, 0, 4092 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_UFS_DEVICE_WLUN), NULL); 4093 if (IS_ERR(hba->sdev_ufs_device)) { 4094 ret = PTR_ERR(hba->sdev_ufs_device); 4095 hba->sdev_ufs_device = NULL; 4096 goto out; 4097 } 4098 scsi_device_put(hba->sdev_ufs_device); 4099 4100 sdev_boot = __scsi_add_device(hba->host, 0, 0, 4101 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_BOOT_WLUN), NULL); 4102 if (IS_ERR(sdev_boot)) { 4103 ret = PTR_ERR(sdev_boot); 4104 goto remove_sdev_ufs_device; 4105 } 4106 scsi_device_put(sdev_boot); 4107 4108 sdev_rpmb = __scsi_add_device(hba->host, 0, 0, 4109 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_RPMB_WLUN), NULL); 4110 if (IS_ERR(sdev_rpmb)) { 4111 ret = PTR_ERR(sdev_rpmb); 4112 goto remove_sdev_boot; 4113 } 4114 scsi_device_put(sdev_rpmb); 4115 goto out; 4116 4117remove_sdev_boot: 4118 scsi_remove_device(sdev_boot); 4119remove_sdev_ufs_device: 4120 scsi_remove_device(hba->sdev_ufs_device); 4121out: 4122 return ret; 4123} 4124 4125/** 4126 * ufshcd_probe_hba - probe hba to detect device and initialize 4127 * @hba: per-adapter instance 4128 * 4129 * Execute link-startup and verify device initialization 4130 */ 4131static int ufshcd_probe_hba(struct ufs_hba *hba) 4132{ 4133 int ret; 4134 4135 ret = ufshcd_link_startup(hba); 4136 if (ret) 4137 goto out; 4138 4139 ufshcd_init_pwr_info(hba); 4140 4141 /* UniPro link is active now */ 4142 ufshcd_set_link_active(hba); 4143 4144 ret = ufshcd_verify_dev_init(hba); 4145 if (ret) 4146 goto out; 4147 4148 ret = ufshcd_complete_dev_init(hba); 4149 if (ret) 4150 goto out; 4151 4152 /* UFS device is also active now */ 4153 ufshcd_set_ufs_dev_active(hba); 4154 ufshcd_force_reset_auto_bkops(hba); 4155 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL; 4156 hba->wlun_dev_clr_ua = true; 4157 4158 if (ufshcd_get_max_pwr_mode(hba)) { 4159 dev_err(hba->dev, 4160 "%s: Failed getting max supported power mode\n", 4161 __func__); 4162 } else { 4163 ret = ufshcd_config_pwr_mode(hba, &hba->max_pwr_info.info); 4164 if (ret) 4165 dev_err(hba->dev, "%s: Failed setting power mode, err = %d\n", 4166 __func__, ret); 4167 } 4168 4169 /* 4170 * If we are in error handling context or in power management callbacks 4171 * context, no need to scan the host 4172 */ 4173 if (!ufshcd_eh_in_progress(hba) && !hba->pm_op_in_progress) { 4174 bool flag; 4175 4176 /* clear any previous UFS device information */ 4177 memset(&hba->dev_info, 0, sizeof(hba->dev_info)); 4178 if (!ufshcd_query_flag(hba, UPIU_QUERY_OPCODE_READ_FLAG, 4179 QUERY_FLAG_IDN_PWR_ON_WPE, &flag)) 4180 hba->dev_info.f_power_on_wp_en = flag; 4181 4182 if (!hba->is_init_prefetch) 4183 ufshcd_init_icc_levels(hba); 4184 4185 /* Add required well known logical units to scsi mid layer */ 4186 if (ufshcd_scsi_add_wlus(hba)) 4187 goto out; 4188 4189 scsi_scan_host(hba->host); 4190 pm_runtime_put_sync(hba->dev); 4191 } 4192 4193 if (!hba->is_init_prefetch) 4194 hba->is_init_prefetch = true; 4195 4196 /* Resume devfreq after UFS device is detected */ 4197 if (ufshcd_is_clkscaling_enabled(hba)) 4198 devfreq_resume_device(hba->devfreq); 4199 4200out: 4201 /* 4202 * If we failed to initialize the device or the device is not 4203 * present, turn off the power/clocks etc. 4204 */ 4205 if (ret && !ufshcd_eh_in_progress(hba) && !hba->pm_op_in_progress) { 4206 pm_runtime_put_sync(hba->dev); 4207 ufshcd_hba_exit(hba); 4208 } 4209 4210 return ret; 4211} 4212 4213/** 4214 * ufshcd_async_scan - asynchronous execution for probing hba 4215 * @data: data pointer to pass to this function 4216 * @cookie: cookie data 4217 */ 4218static void ufshcd_async_scan(void *data, async_cookie_t cookie) 4219{ 4220 struct ufs_hba *hba = (struct ufs_hba *)data; 4221 4222 ufshcd_probe_hba(hba); 4223} 4224 4225static struct scsi_host_template ufshcd_driver_template = { 4226 .module = THIS_MODULE, 4227 .name = UFSHCD, 4228 .proc_name = UFSHCD, 4229 .queuecommand = ufshcd_queuecommand, 4230 .slave_alloc = ufshcd_slave_alloc, 4231 .slave_configure = ufshcd_slave_configure, 4232 .slave_destroy = ufshcd_slave_destroy, 4233 .change_queue_depth = ufshcd_change_queue_depth, 4234 .eh_abort_handler = ufshcd_abort, 4235 .eh_device_reset_handler = ufshcd_eh_device_reset_handler, 4236 .eh_host_reset_handler = ufshcd_eh_host_reset_handler, 4237 .this_id = -1, 4238 .sg_tablesize = SG_ALL, 4239 .cmd_per_lun = UFSHCD_CMD_PER_LUN, 4240 .can_queue = UFSHCD_CAN_QUEUE, 4241 .max_host_blocked = 1, 4242}; 4243 4244static int ufshcd_config_vreg_load(struct device *dev, struct ufs_vreg *vreg, 4245 int ua) 4246{ 4247 int ret = 0; 4248 struct regulator *reg = vreg->reg; 4249 const char *name = vreg->name; 4250 4251 BUG_ON(!vreg); 4252 4253 ret = regulator_set_optimum_mode(reg, ua); 4254 if (ret >= 0) { 4255 /* 4256 * regulator_set_optimum_mode() returns new regulator 4257 * mode upon success. 4258 */ 4259 ret = 0; 4260 } else { 4261 dev_err(dev, "%s: %s set optimum mode(ua=%d) failed, err=%d\n", 4262 __func__, name, ua, ret); 4263 } 4264 4265 return ret; 4266} 4267 4268static inline int ufshcd_config_vreg_lpm(struct ufs_hba *hba, 4269 struct ufs_vreg *vreg) 4270{ 4271 if (!vreg) 4272 return 0; 4273 4274 return ufshcd_config_vreg_load(hba->dev, vreg, UFS_VREG_LPM_LOAD_UA); 4275} 4276 4277static inline int ufshcd_config_vreg_hpm(struct ufs_hba *hba, 4278 struct ufs_vreg *vreg) 4279{ 4280 if (!vreg) 4281 return 0; 4282 4283 return ufshcd_config_vreg_load(hba->dev, vreg, vreg->max_uA); 4284} 4285 4286static int ufshcd_config_vreg(struct device *dev, 4287 struct ufs_vreg *vreg, bool on) 4288{ 4289 int ret = 0; 4290 struct regulator *reg = vreg->reg; 4291 const char *name = vreg->name; 4292 int min_uV, uA_load; 4293 4294 BUG_ON(!vreg); 4295 4296 if (regulator_count_voltages(reg) > 0) { 4297 min_uV = on ? vreg->min_uV : 0; 4298 ret = regulator_set_voltage(reg, min_uV, vreg->max_uV); 4299 if (ret) { 4300 dev_err(dev, "%s: %s set voltage failed, err=%d\n", 4301 __func__, name, ret); 4302 goto out; 4303 } 4304 4305 uA_load = on ? vreg->max_uA : 0; 4306 ret = ufshcd_config_vreg_load(dev, vreg, uA_load); 4307 if (ret) 4308 goto out; 4309 } 4310out: 4311 return ret; 4312} 4313 4314static int ufshcd_enable_vreg(struct device *dev, struct ufs_vreg *vreg) 4315{ 4316 int ret = 0; 4317 4318 if (!vreg || vreg->enabled) 4319 goto out; 4320 4321 ret = ufshcd_config_vreg(dev, vreg, true); 4322 if (!ret) 4323 ret = regulator_enable(vreg->reg); 4324 4325 if (!ret) 4326 vreg->enabled = true; 4327 else 4328 dev_err(dev, "%s: %s enable failed, err=%d\n", 4329 __func__, vreg->name, ret); 4330out: 4331 return ret; 4332} 4333 4334static int ufshcd_disable_vreg(struct device *dev, struct ufs_vreg *vreg) 4335{ 4336 int ret = 0; 4337 4338 if (!vreg || !vreg->enabled) 4339 goto out; 4340 4341 ret = regulator_disable(vreg->reg); 4342 4343 if (!ret) { 4344 /* ignore errors on applying disable config */ 4345 ufshcd_config_vreg(dev, vreg, false); 4346 vreg->enabled = false; 4347 } else { 4348 dev_err(dev, "%s: %s disable failed, err=%d\n", 4349 __func__, vreg->name, ret); 4350 } 4351out: 4352 return ret; 4353} 4354 4355static int ufshcd_setup_vreg(struct ufs_hba *hba, bool on) 4356{ 4357 int ret = 0; 4358 struct device *dev = hba->dev; 4359 struct ufs_vreg_info *info = &hba->vreg_info; 4360 4361 if (!info) 4362 goto out; 4363 4364 ret = ufshcd_toggle_vreg(dev, info->vcc, on); 4365 if (ret) 4366 goto out; 4367 4368 ret = ufshcd_toggle_vreg(dev, info->vccq, on); 4369 if (ret) 4370 goto out; 4371 4372 ret = ufshcd_toggle_vreg(dev, info->vccq2, on); 4373 if (ret) 4374 goto out; 4375 4376out: 4377 if (ret) { 4378 ufshcd_toggle_vreg(dev, info->vccq2, false); 4379 ufshcd_toggle_vreg(dev, info->vccq, false); 4380 ufshcd_toggle_vreg(dev, info->vcc, false); 4381 } 4382 return ret; 4383} 4384 4385static int ufshcd_setup_hba_vreg(struct ufs_hba *hba, bool on) 4386{ 4387 struct ufs_vreg_info *info = &hba->vreg_info; 4388 4389 if (info) 4390 return ufshcd_toggle_vreg(hba->dev, info->vdd_hba, on); 4391 4392 return 0; 4393} 4394 4395static int ufshcd_get_vreg(struct device *dev, struct ufs_vreg *vreg) 4396{ 4397 int ret = 0; 4398 4399 if (!vreg) 4400 goto out; 4401 4402 vreg->reg = devm_regulator_get(dev, vreg->name); 4403 if (IS_ERR(vreg->reg)) { 4404 ret = PTR_ERR(vreg->reg); 4405 dev_err(dev, "%s: %s get failed, err=%d\n", 4406 __func__, vreg->name, ret); 4407 } 4408out: 4409 return ret; 4410} 4411 4412static int ufshcd_init_vreg(struct ufs_hba *hba) 4413{ 4414 int ret = 0; 4415 struct device *dev = hba->dev; 4416 struct ufs_vreg_info *info = &hba->vreg_info; 4417 4418 if (!info) 4419 goto out; 4420 4421 ret = ufshcd_get_vreg(dev, info->vcc); 4422 if (ret) 4423 goto out; 4424 4425 ret = ufshcd_get_vreg(dev, info->vccq); 4426 if (ret) 4427 goto out; 4428 4429 ret = ufshcd_get_vreg(dev, info->vccq2); 4430out: 4431 return ret; 4432} 4433 4434static int ufshcd_init_hba_vreg(struct ufs_hba *hba) 4435{ 4436 struct ufs_vreg_info *info = &hba->vreg_info; 4437 4438 if (info) 4439 return ufshcd_get_vreg(hba->dev, info->vdd_hba); 4440 4441 return 0; 4442} 4443 4444static int __ufshcd_setup_clocks(struct ufs_hba *hba, bool on, 4445 bool skip_ref_clk) 4446{ 4447 int ret = 0; 4448 struct ufs_clk_info *clki; 4449 struct list_head *head = &hba->clk_list_head; 4450 unsigned long flags; 4451 4452 if (!head || list_empty(head)) 4453 goto out; 4454 4455 list_for_each_entry(clki, head, list) { 4456 if (!IS_ERR_OR_NULL(clki->clk)) { 4457 if (skip_ref_clk && !strcmp(clki->name, "ref_clk")) 4458 continue; 4459 4460 if (on && !clki->enabled) { 4461 ret = clk_prepare_enable(clki->clk); 4462 if (ret) { 4463 dev_err(hba->dev, "%s: %s prepare enable failed, %d\n", 4464 __func__, clki->name, ret); 4465 goto out; 4466 } 4467 } else if (!on && clki->enabled) { 4468 clk_disable_unprepare(clki->clk); 4469 } 4470 clki->enabled = on; 4471 dev_dbg(hba->dev, "%s: clk: %s %sabled\n", __func__, 4472 clki->name, on ? "en" : "dis"); 4473 } 4474 } 4475 4476 if (hba->vops && hba->vops->setup_clocks) 4477 ret = hba->vops->setup_clocks(hba, on); 4478out: 4479 if (ret) { 4480 list_for_each_entry(clki, head, list) { 4481 if (!IS_ERR_OR_NULL(clki->clk) && clki->enabled) 4482 clk_disable_unprepare(clki->clk); 4483 } 4484 } else if (on) { 4485 spin_lock_irqsave(hba->host->host_lock, flags); 4486 hba->clk_gating.state = CLKS_ON; 4487 spin_unlock_irqrestore(hba->host->host_lock, flags); 4488 } 4489 return ret; 4490} 4491 4492static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on) 4493{ 4494 return __ufshcd_setup_clocks(hba, on, false); 4495} 4496 4497static int ufshcd_init_clocks(struct ufs_hba *hba) 4498{ 4499 int ret = 0; 4500 struct ufs_clk_info *clki; 4501 struct device *dev = hba->dev; 4502 struct list_head *head = &hba->clk_list_head; 4503 4504 if (!head || list_empty(head)) 4505 goto out; 4506 4507 list_for_each_entry(clki, head, list) { 4508 if (!clki->name) 4509 continue; 4510 4511 clki->clk = devm_clk_get(dev, clki->name); 4512 if (IS_ERR(clki->clk)) { 4513 ret = PTR_ERR(clki->clk); 4514 dev_err(dev, "%s: %s clk get failed, %d\n", 4515 __func__, clki->name, ret); 4516 goto out; 4517 } 4518 4519 if (clki->max_freq) { 4520 ret = clk_set_rate(clki->clk, clki->max_freq); 4521 if (ret) { 4522 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n", 4523 __func__, clki->name, 4524 clki->max_freq, ret); 4525 goto out; 4526 } 4527 clki->curr_freq = clki->max_freq; 4528 } 4529 dev_dbg(dev, "%s: clk: %s, rate: %lu\n", __func__, 4530 clki->name, clk_get_rate(clki->clk)); 4531 } 4532out: 4533 return ret; 4534} 4535 4536static int ufshcd_variant_hba_init(struct ufs_hba *hba) 4537{ 4538 int err = 0; 4539 4540 if (!hba->vops) 4541 goto out; 4542 4543 if (hba->vops->init) { 4544 err = hba->vops->init(hba); 4545 if (err) 4546 goto out; 4547 } 4548 4549 if (hba->vops->setup_regulators) { 4550 err = hba->vops->setup_regulators(hba, true); 4551 if (err) 4552 goto out_exit; 4553 } 4554 4555 goto out; 4556 4557out_exit: 4558 if (hba->vops->exit) 4559 hba->vops->exit(hba); 4560out: 4561 if (err) 4562 dev_err(hba->dev, "%s: variant %s init failed err %d\n", 4563 __func__, hba->vops ? hba->vops->name : "", err); 4564 return err; 4565} 4566 4567static void ufshcd_variant_hba_exit(struct ufs_hba *hba) 4568{ 4569 if (!hba->vops) 4570 return; 4571 4572 if (hba->vops->setup_clocks) 4573 hba->vops->setup_clocks(hba, false); 4574 4575 if (hba->vops->setup_regulators) 4576 hba->vops->setup_regulators(hba, false); 4577 4578 if (hba->vops->exit) 4579 hba->vops->exit(hba); 4580} 4581 4582static int ufshcd_hba_init(struct ufs_hba *hba) 4583{ 4584 int err; 4585 4586 /* 4587 * Handle host controller power separately from the UFS device power 4588 * rails as it will help controlling the UFS host controller power 4589 * collapse easily which is different than UFS device power collapse. 4590 * Also, enable the host controller power before we go ahead with rest 4591 * of the initialization here. 4592 */ 4593 err = ufshcd_init_hba_vreg(hba); 4594 if (err) 4595 goto out; 4596 4597 err = ufshcd_setup_hba_vreg(hba, true); 4598 if (err) 4599 goto out; 4600 4601 err = ufshcd_init_clocks(hba); 4602 if (err) 4603 goto out_disable_hba_vreg; 4604 4605 err = ufshcd_setup_clocks(hba, true); 4606 if (err) 4607 goto out_disable_hba_vreg; 4608 4609 err = ufshcd_init_vreg(hba); 4610 if (err) 4611 goto out_disable_clks; 4612 4613 err = ufshcd_setup_vreg(hba, true); 4614 if (err) 4615 goto out_disable_clks; 4616 4617 err = ufshcd_variant_hba_init(hba); 4618 if (err) 4619 goto out_disable_vreg; 4620 4621 hba->is_powered = true; 4622 goto out; 4623 4624out_disable_vreg: 4625 ufshcd_setup_vreg(hba, false); 4626out_disable_clks: 4627 ufshcd_setup_clocks(hba, false); 4628out_disable_hba_vreg: 4629 ufshcd_setup_hba_vreg(hba, false); 4630out: 4631 return err; 4632} 4633 4634static void ufshcd_hba_exit(struct ufs_hba *hba) 4635{ 4636 if (hba->is_powered) { 4637 ufshcd_variant_hba_exit(hba); 4638 ufshcd_setup_vreg(hba, false); 4639 ufshcd_setup_clocks(hba, false); 4640 ufshcd_setup_hba_vreg(hba, false); 4641 hba->is_powered = false; 4642 } 4643} 4644 4645static int 4646ufshcd_send_request_sense(struct ufs_hba *hba, struct scsi_device *sdp) 4647{ 4648 unsigned char cmd[6] = {REQUEST_SENSE, 4649 0, 4650 0, 4651 0, 4652 SCSI_SENSE_BUFFERSIZE, 4653 0}; 4654 char *buffer; 4655 int ret; 4656 4657 buffer = kzalloc(SCSI_SENSE_BUFFERSIZE, GFP_KERNEL); 4658 if (!buffer) { 4659 ret = -ENOMEM; 4660 goto out; 4661 } 4662 4663 ret = scsi_execute_req_flags(sdp, cmd, DMA_FROM_DEVICE, buffer, 4664 SCSI_SENSE_BUFFERSIZE, NULL, 4665 msecs_to_jiffies(1000), 3, NULL, REQ_PM); 4666 if (ret) 4667 pr_err("%s: failed with err %d\n", __func__, ret); 4668 4669 kfree(buffer); 4670out: 4671 return ret; 4672} 4673 4674/** 4675 * ufshcd_set_dev_pwr_mode - sends START STOP UNIT command to set device 4676 * power mode 4677 * @hba: per adapter instance 4678 * @pwr_mode: device power mode to set 4679 * 4680 * Returns 0 if requested power mode is set successfully 4681 * Returns non-zero if failed to set the requested power mode 4682 */ 4683static int ufshcd_set_dev_pwr_mode(struct ufs_hba *hba, 4684 enum ufs_dev_pwr_mode pwr_mode) 4685{ 4686 unsigned char cmd[6] = { START_STOP }; 4687 struct scsi_sense_hdr sshdr; 4688 struct scsi_device *sdp; 4689 unsigned long flags; 4690 int ret; 4691 4692 spin_lock_irqsave(hba->host->host_lock, flags); 4693 sdp = hba->sdev_ufs_device; 4694 if (sdp) { 4695 ret = scsi_device_get(sdp); 4696 if (!ret && !scsi_device_online(sdp)) { 4697 ret = -ENODEV; 4698 scsi_device_put(sdp); 4699 } 4700 } else { 4701 ret = -ENODEV; 4702 } 4703 spin_unlock_irqrestore(hba->host->host_lock, flags); 4704 4705 if (ret) 4706 return ret; 4707 4708 /* 4709 * If scsi commands fail, the scsi mid-layer schedules scsi error- 4710 * handling, which would wait for host to be resumed. Since we know 4711 * we are functional while we are here, skip host resume in error 4712 * handling context. 4713 */ 4714 hba->host->eh_noresume = 1; 4715 if (hba->wlun_dev_clr_ua) { 4716 ret = ufshcd_send_request_sense(hba, sdp); 4717 if (ret) 4718 goto out; 4719 /* Unit attention condition is cleared now */ 4720 hba->wlun_dev_clr_ua = false; 4721 } 4722 4723 cmd[4] = pwr_mode << 4; 4724 4725 /* 4726 * Current function would be generally called from the power management 4727 * callbacks hence set the REQ_PM flag so that it doesn't resume the 4728 * already suspended childs. 4729 */ 4730 ret = scsi_execute_req_flags(sdp, cmd, DMA_NONE, NULL, 0, &sshdr, 4731 START_STOP_TIMEOUT, 0, NULL, REQ_PM); 4732 if (ret) { 4733 sdev_printk(KERN_WARNING, sdp, 4734 "START_STOP failed for power mode: %d\n", pwr_mode); 4735 scsi_show_result(ret); 4736 if (driver_byte(ret) & DRIVER_SENSE) { 4737 scsi_show_sense_hdr(&sshdr); 4738 scsi_show_extd_sense(sshdr.asc, sshdr.ascq); 4739 } 4740 } 4741 4742 if (!ret) 4743 hba->curr_dev_pwr_mode = pwr_mode; 4744out: 4745 scsi_device_put(sdp); 4746 hba->host->eh_noresume = 0; 4747 return ret; 4748} 4749 4750static int ufshcd_link_state_transition(struct ufs_hba *hba, 4751 enum uic_link_state req_link_state, 4752 int check_for_bkops) 4753{ 4754 int ret = 0; 4755 4756 if (req_link_state == hba->uic_link_state) 4757 return 0; 4758 4759 if (req_link_state == UIC_LINK_HIBERN8_STATE) { 4760 ret = ufshcd_uic_hibern8_enter(hba); 4761 if (!ret) 4762 ufshcd_set_link_hibern8(hba); 4763 else 4764 goto out; 4765 } 4766 /* 4767 * If autobkops is enabled, link can't be turned off because 4768 * turning off the link would also turn off the device. 4769 */ 4770 else if ((req_link_state == UIC_LINK_OFF_STATE) && 4771 (!check_for_bkops || (check_for_bkops && 4772 !hba->auto_bkops_enabled))) { 4773 /* 4774 * Change controller state to "reset state" which 4775 * should also put the link in off/reset state 4776 */ 4777 ufshcd_hba_stop(hba); 4778 /* 4779 * TODO: Check if we need any delay to make sure that 4780 * controller is reset 4781 */ 4782 ufshcd_set_link_off(hba); 4783 } 4784 4785out: 4786 return ret; 4787} 4788 4789static void ufshcd_vreg_set_lpm(struct ufs_hba *hba) 4790{ 4791 /* 4792 * If UFS device is either in UFS_Sleep turn off VCC rail to save some 4793 * power. 4794 * 4795 * If UFS device and link is in OFF state, all power supplies (VCC, 4796 * VCCQ, VCCQ2) can be turned off if power on write protect is not 4797 * required. If UFS link is inactive (Hibern8 or OFF state) and device 4798 * is in sleep state, put VCCQ & VCCQ2 rails in LPM mode. 4799 * 4800 * Ignore the error returned by ufshcd_toggle_vreg() as device is anyway 4801 * in low power state which would save some power. 4802 */ 4803 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) && 4804 !hba->dev_info.is_lu_power_on_wp) { 4805 ufshcd_setup_vreg(hba, false); 4806 } else if (!ufshcd_is_ufs_dev_active(hba)) { 4807 ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false); 4808 if (!ufshcd_is_link_active(hba)) { 4809 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq); 4810 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq2); 4811 } 4812 } 4813} 4814 4815static int ufshcd_vreg_set_hpm(struct ufs_hba *hba) 4816{ 4817 int ret = 0; 4818 4819 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) && 4820 !hba->dev_info.is_lu_power_on_wp) { 4821 ret = ufshcd_setup_vreg(hba, true); 4822 } else if (!ufshcd_is_ufs_dev_active(hba)) { 4823 ret = ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, true); 4824 if (!ret && !ufshcd_is_link_active(hba)) { 4825 ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq); 4826 if (ret) 4827 goto vcc_disable; 4828 ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq2); 4829 if (ret) 4830 goto vccq_lpm; 4831 } 4832 } 4833 goto out; 4834 4835vccq_lpm: 4836 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq); 4837vcc_disable: 4838 ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false); 4839out: 4840 return ret; 4841} 4842 4843static void ufshcd_hba_vreg_set_lpm(struct ufs_hba *hba) 4844{ 4845 if (ufshcd_is_link_off(hba)) 4846 ufshcd_setup_hba_vreg(hba, false); 4847} 4848 4849static void ufshcd_hba_vreg_set_hpm(struct ufs_hba *hba) 4850{ 4851 if (ufshcd_is_link_off(hba)) 4852 ufshcd_setup_hba_vreg(hba, true); 4853} 4854 4855/** 4856 * ufshcd_suspend - helper function for suspend operations 4857 * @hba: per adapter instance 4858 * @pm_op: desired low power operation type 4859 * 4860 * This function will try to put the UFS device and link into low power 4861 * mode based on the "rpm_lvl" (Runtime PM level) or "spm_lvl" 4862 * (System PM level). 4863 * 4864 * If this function is called during shutdown, it will make sure that 4865 * both UFS device and UFS link is powered off. 4866 * 4867 * NOTE: UFS device & link must be active before we enter in this function. 4868 * 4869 * Returns 0 for success and non-zero for failure 4870 */ 4871static int ufshcd_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op) 4872{ 4873 int ret = 0; 4874 enum ufs_pm_level pm_lvl; 4875 enum ufs_dev_pwr_mode req_dev_pwr_mode; 4876 enum uic_link_state req_link_state; 4877 4878 hba->pm_op_in_progress = 1; 4879 if (!ufshcd_is_shutdown_pm(pm_op)) { 4880 pm_lvl = ufshcd_is_runtime_pm(pm_op) ? 4881 hba->rpm_lvl : hba->spm_lvl; 4882 req_dev_pwr_mode = ufs_get_pm_lvl_to_dev_pwr_mode(pm_lvl); 4883 req_link_state = ufs_get_pm_lvl_to_link_pwr_state(pm_lvl); 4884 } else { 4885 req_dev_pwr_mode = UFS_POWERDOWN_PWR_MODE; 4886 req_link_state = UIC_LINK_OFF_STATE; 4887 } 4888 4889 /* 4890 * If we can't transition into any of the low power modes 4891 * just gate the clocks. 4892 */ 4893 ufshcd_hold(hba, false); 4894 hba->clk_gating.is_suspended = true; 4895 4896 if (req_dev_pwr_mode == UFS_ACTIVE_PWR_MODE && 4897 req_link_state == UIC_LINK_ACTIVE_STATE) { 4898 goto disable_clks; 4899 } 4900 4901 if ((req_dev_pwr_mode == hba->curr_dev_pwr_mode) && 4902 (req_link_state == hba->uic_link_state)) 4903 goto out; 4904 4905 /* UFS device & link must be active before we enter in this function */ 4906 if (!ufshcd_is_ufs_dev_active(hba) || !ufshcd_is_link_active(hba)) { 4907 ret = -EINVAL; 4908 goto out; 4909 } 4910 4911 if (ufshcd_is_runtime_pm(pm_op)) { 4912 if (ufshcd_can_autobkops_during_suspend(hba)) { 4913 /* 4914 * The device is idle with no requests in the queue, 4915 * allow background operations if bkops status shows 4916 * that performance might be impacted. 4917 */ 4918 ret = ufshcd_urgent_bkops(hba); 4919 if (ret) 4920 goto enable_gating; 4921 } else { 4922 /* make sure that auto bkops is disabled */ 4923 ufshcd_disable_auto_bkops(hba); 4924 } 4925 } 4926 4927 if ((req_dev_pwr_mode != hba->curr_dev_pwr_mode) && 4928 ((ufshcd_is_runtime_pm(pm_op) && !hba->auto_bkops_enabled) || 4929 !ufshcd_is_runtime_pm(pm_op))) { 4930 /* ensure that bkops is disabled */ 4931 ufshcd_disable_auto_bkops(hba); 4932 ret = ufshcd_set_dev_pwr_mode(hba, req_dev_pwr_mode); 4933 if (ret) 4934 goto enable_gating; 4935 } 4936 4937 ret = ufshcd_link_state_transition(hba, req_link_state, 1); 4938 if (ret) 4939 goto set_dev_active; 4940 4941 ufshcd_vreg_set_lpm(hba); 4942 4943disable_clks: 4944 /* 4945 * The clock scaling needs access to controller registers. Hence, Wait 4946 * for pending clock scaling work to be done before clocks are 4947 * turned off. 4948 */ 4949 if (ufshcd_is_clkscaling_enabled(hba)) { 4950 devfreq_suspend_device(hba->devfreq); 4951 hba->clk_scaling.window_start_t = 0; 4952 } 4953 /* 4954 * Call vendor specific suspend callback. As these callbacks may access 4955 * vendor specific host controller register space call them before the 4956 * host clocks are ON. 4957 */ 4958 if (hba->vops && hba->vops->suspend) { 4959 ret = hba->vops->suspend(hba, pm_op); 4960 if (ret) 4961 goto set_link_active; 4962 } 4963 4964 if (hba->vops && hba->vops->setup_clocks) { 4965 ret = hba->vops->setup_clocks(hba, false); 4966 if (ret) 4967 goto vops_resume; 4968 } 4969 4970 if (!ufshcd_is_link_active(hba)) 4971 ufshcd_setup_clocks(hba, false); 4972 else 4973 /* If link is active, device ref_clk can't be switched off */ 4974 __ufshcd_setup_clocks(hba, false, true); 4975 4976 hba->clk_gating.state = CLKS_OFF; 4977 /* 4978 * Disable the host irq as host controller as there won't be any 4979 * host controller trasanction expected till resume. 4980 */ 4981 ufshcd_disable_irq(hba); 4982 /* Put the host controller in low power mode if possible */ 4983 ufshcd_hba_vreg_set_lpm(hba); 4984 goto out; 4985 4986vops_resume: 4987 if (hba->vops && hba->vops->resume) 4988 hba->vops->resume(hba, pm_op); 4989set_link_active: 4990 ufshcd_vreg_set_hpm(hba); 4991 if (ufshcd_is_link_hibern8(hba) && !ufshcd_uic_hibern8_exit(hba)) 4992 ufshcd_set_link_active(hba); 4993 else if (ufshcd_is_link_off(hba)) 4994 ufshcd_host_reset_and_restore(hba); 4995set_dev_active: 4996 if (!ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE)) 4997 ufshcd_disable_auto_bkops(hba); 4998enable_gating: 4999 hba->clk_gating.is_suspended = false; 5000 ufshcd_release(hba); 5001out: 5002 hba->pm_op_in_progress = 0; 5003 return ret; 5004} 5005 5006/** 5007 * ufshcd_resume - helper function for resume operations 5008 * @hba: per adapter instance 5009 * @pm_op: runtime PM or system PM 5010 * 5011 * This function basically brings the UFS device, UniPro link and controller 5012 * to active state. 5013 * 5014 * Returns 0 for success and non-zero for failure 5015 */ 5016static int ufshcd_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op) 5017{ 5018 int ret; 5019 enum uic_link_state old_link_state; 5020 5021 hba->pm_op_in_progress = 1; 5022 old_link_state = hba->uic_link_state; 5023 5024 ufshcd_hba_vreg_set_hpm(hba); 5025 /* Make sure clocks are enabled before accessing controller */ 5026 ret = ufshcd_setup_clocks(hba, true); 5027 if (ret) 5028 goto out; 5029 5030 /* enable the host irq as host controller would be active soon */ 5031 ret = ufshcd_enable_irq(hba); 5032 if (ret) 5033 goto disable_irq_and_vops_clks; 5034 5035 ret = ufshcd_vreg_set_hpm(hba); 5036 if (ret) 5037 goto disable_irq_and_vops_clks; 5038 5039 /* 5040 * Call vendor specific resume callback. As these callbacks may access 5041 * vendor specific host controller register space call them when the 5042 * host clocks are ON. 5043 */ 5044 if (hba->vops && hba->vops->resume) { 5045 ret = hba->vops->resume(hba, pm_op); 5046 if (ret) 5047 goto disable_vreg; 5048 } 5049 5050 if (ufshcd_is_link_hibern8(hba)) { 5051 ret = ufshcd_uic_hibern8_exit(hba); 5052 if (!ret) 5053 ufshcd_set_link_active(hba); 5054 else 5055 goto vendor_suspend; 5056 } else if (ufshcd_is_link_off(hba)) { 5057 ret = ufshcd_host_reset_and_restore(hba); 5058 /* 5059 * ufshcd_host_reset_and_restore() should have already 5060 * set the link state as active 5061 */ 5062 if (ret || !ufshcd_is_link_active(hba)) 5063 goto vendor_suspend; 5064 } 5065 5066 if (!ufshcd_is_ufs_dev_active(hba)) { 5067 ret = ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE); 5068 if (ret) 5069 goto set_old_link_state; 5070 } 5071 5072 /* 5073 * If BKOPs operations are urgently needed at this moment then 5074 * keep auto-bkops enabled or else disable it. 5075 */ 5076 ufshcd_urgent_bkops(hba); 5077 hba->clk_gating.is_suspended = false; 5078 5079 if (ufshcd_is_clkscaling_enabled(hba)) 5080 devfreq_resume_device(hba->devfreq); 5081 5082 /* Schedule clock gating in case of no access to UFS device yet */ 5083 ufshcd_release(hba); 5084 goto out; 5085 5086set_old_link_state: 5087 ufshcd_link_state_transition(hba, old_link_state, 0); 5088vendor_suspend: 5089 if (hba->vops && hba->vops->suspend) 5090 hba->vops->suspend(hba, pm_op); 5091disable_vreg: 5092 ufshcd_vreg_set_lpm(hba); 5093disable_irq_and_vops_clks: 5094 ufshcd_disable_irq(hba); 5095 ufshcd_setup_clocks(hba, false); 5096out: 5097 hba->pm_op_in_progress = 0; 5098 return ret; 5099} 5100 5101/** 5102 * ufshcd_system_suspend - system suspend routine 5103 * @hba: per adapter instance 5104 * @pm_op: runtime PM or system PM 5105 * 5106 * Check the description of ufshcd_suspend() function for more details. 5107 * 5108 * Returns 0 for success and non-zero for failure 5109 */ 5110int ufshcd_system_suspend(struct ufs_hba *hba) 5111{ 5112 int ret = 0; 5113 5114 if (!hba || !hba->is_powered) 5115 return 0; 5116 5117 if (pm_runtime_suspended(hba->dev)) { 5118 if (hba->rpm_lvl == hba->spm_lvl) 5119 /* 5120 * There is possibility that device may still be in 5121 * active state during the runtime suspend. 5122 */ 5123 if ((ufs_get_pm_lvl_to_dev_pwr_mode(hba->spm_lvl) == 5124 hba->curr_dev_pwr_mode) && !hba->auto_bkops_enabled) 5125 goto out; 5126 5127 /* 5128 * UFS device and/or UFS link low power states during runtime 5129 * suspend seems to be different than what is expected during 5130 * system suspend. Hence runtime resume the devic & link and 5131 * let the system suspend low power states to take effect. 5132 * TODO: If resume takes longer time, we might have optimize 5133 * it in future by not resuming everything if possible. 5134 */ 5135 ret = ufshcd_runtime_resume(hba); 5136 if (ret) 5137 goto out; 5138 } 5139 5140 ret = ufshcd_suspend(hba, UFS_SYSTEM_PM); 5141out: 5142 if (!ret) 5143 hba->is_sys_suspended = true; 5144 return ret; 5145} 5146EXPORT_SYMBOL(ufshcd_system_suspend); 5147 5148/** 5149 * ufshcd_system_resume - system resume routine 5150 * @hba: per adapter instance 5151 * 5152 * Returns 0 for success and non-zero for failure 5153 */ 5154 5155int ufshcd_system_resume(struct ufs_hba *hba) 5156{ 5157 if (!hba || !hba->is_powered || pm_runtime_suspended(hba->dev)) 5158 /* 5159 * Let the runtime resume take care of resuming 5160 * if runtime suspended. 5161 */ 5162 return 0; 5163 5164 return ufshcd_resume(hba, UFS_SYSTEM_PM); 5165} 5166EXPORT_SYMBOL(ufshcd_system_resume); 5167 5168/** 5169 * ufshcd_runtime_suspend - runtime suspend routine 5170 * @hba: per adapter instance 5171 * 5172 * Check the description of ufshcd_suspend() function for more details. 5173 * 5174 * Returns 0 for success and non-zero for failure 5175 */ 5176int ufshcd_runtime_suspend(struct ufs_hba *hba) 5177{ 5178 if (!hba || !hba->is_powered) 5179 return 0; 5180 5181 return ufshcd_suspend(hba, UFS_RUNTIME_PM); 5182} 5183EXPORT_SYMBOL(ufshcd_runtime_suspend); 5184 5185/** 5186 * ufshcd_runtime_resume - runtime resume routine 5187 * @hba: per adapter instance 5188 * 5189 * This function basically brings the UFS device, UniPro link and controller 5190 * to active state. Following operations are done in this function: 5191 * 5192 * 1. Turn on all the controller related clocks 5193 * 2. Bring the UniPro link out of Hibernate state 5194 * 3. If UFS device is in sleep state, turn ON VCC rail and bring the UFS device 5195 * to active state. 5196 * 4. If auto-bkops is enabled on the device, disable it. 5197 * 5198 * So following would be the possible power state after this function return 5199 * successfully: 5200 * S1: UFS device in Active state with VCC rail ON 5201 * UniPro link in Active state 5202 * All the UFS/UniPro controller clocks are ON 5203 * 5204 * Returns 0 for success and non-zero for failure 5205 */ 5206int ufshcd_runtime_resume(struct ufs_hba *hba) 5207{ 5208 if (!hba || !hba->is_powered) 5209 return 0; 5210 else 5211 return ufshcd_resume(hba, UFS_RUNTIME_PM); 5212} 5213EXPORT_SYMBOL(ufshcd_runtime_resume); 5214 5215int ufshcd_runtime_idle(struct ufs_hba *hba) 5216{ 5217 return 0; 5218} 5219EXPORT_SYMBOL(ufshcd_runtime_idle); 5220 5221/** 5222 * ufshcd_shutdown - shutdown routine 5223 * @hba: per adapter instance 5224 * 5225 * This function would power off both UFS device and UFS link. 5226 * 5227 * Returns 0 always to allow force shutdown even in case of errors. 5228 */ 5229int ufshcd_shutdown(struct ufs_hba *hba) 5230{ 5231 int ret = 0; 5232 5233 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba)) 5234 goto out; 5235 5236 if (pm_runtime_suspended(hba->dev)) { 5237 ret = ufshcd_runtime_resume(hba); 5238 if (ret) 5239 goto out; 5240 } 5241 5242 ret = ufshcd_suspend(hba, UFS_SHUTDOWN_PM); 5243out: 5244 if (ret) 5245 dev_err(hba->dev, "%s failed, err %d\n", __func__, ret); 5246 /* allow force shutdown even in case of errors */ 5247 return 0; 5248} 5249EXPORT_SYMBOL(ufshcd_shutdown); 5250 5251/** 5252 * ufshcd_remove - de-allocate SCSI host and host memory space 5253 * data structure memory 5254 * @hba - per adapter instance 5255 */ 5256void ufshcd_remove(struct ufs_hba *hba) 5257{ 5258 scsi_remove_host(hba->host); 5259 /* disable interrupts */ 5260 ufshcd_disable_intr(hba, hba->intr_mask); 5261 ufshcd_hba_stop(hba); 5262 5263 scsi_host_put(hba->host); 5264 5265 ufshcd_exit_clk_gating(hba); 5266 if (ufshcd_is_clkscaling_enabled(hba)) 5267 devfreq_remove_device(hba->devfreq); 5268 ufshcd_hba_exit(hba); 5269} 5270EXPORT_SYMBOL_GPL(ufshcd_remove); 5271 5272/** 5273 * ufshcd_set_dma_mask - Set dma mask based on the controller 5274 * addressing capability 5275 * @hba: per adapter instance 5276 * 5277 * Returns 0 for success, non-zero for failure 5278 */ 5279static int ufshcd_set_dma_mask(struct ufs_hba *hba) 5280{ 5281 if (hba->capabilities & MASK_64_ADDRESSING_SUPPORT) { 5282 if (!dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(64))) 5283 return 0; 5284 } 5285 return dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(32)); 5286} 5287 5288/** 5289 * ufshcd_alloc_host - allocate Host Bus Adapter (HBA) 5290 * @dev: pointer to device handle 5291 * @hba_handle: driver private handle 5292 * Returns 0 on success, non-zero value on failure 5293 */ 5294int ufshcd_alloc_host(struct device *dev, struct ufs_hba **hba_handle) 5295{ 5296 struct Scsi_Host *host; 5297 struct ufs_hba *hba; 5298 int err = 0; 5299 5300 if (!dev) { 5301 dev_err(dev, 5302 "Invalid memory reference for dev is NULL\n"); 5303 err = -ENODEV; 5304 goto out_error; 5305 } 5306 5307 host = scsi_host_alloc(&ufshcd_driver_template, 5308 sizeof(struct ufs_hba)); 5309 if (!host) { 5310 dev_err(dev, "scsi_host_alloc failed\n"); 5311 err = -ENOMEM; 5312 goto out_error; 5313 } 5314 hba = shost_priv(host); 5315 hba->host = host; 5316 hba->dev = dev; 5317 *hba_handle = hba; 5318 5319out_error: 5320 return err; 5321} 5322EXPORT_SYMBOL(ufshcd_alloc_host); 5323 5324static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up) 5325{ 5326 int ret = 0; 5327 struct ufs_clk_info *clki; 5328 struct list_head *head = &hba->clk_list_head; 5329 5330 if (!head || list_empty(head)) 5331 goto out; 5332 5333 list_for_each_entry(clki, head, list) { 5334 if (!IS_ERR_OR_NULL(clki->clk)) { 5335 if (scale_up && clki->max_freq) { 5336 if (clki->curr_freq == clki->max_freq) 5337 continue; 5338 ret = clk_set_rate(clki->clk, clki->max_freq); 5339 if (ret) { 5340 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n", 5341 __func__, clki->name, 5342 clki->max_freq, ret); 5343 break; 5344 } 5345 clki->curr_freq = clki->max_freq; 5346 5347 } else if (!scale_up && clki->min_freq) { 5348 if (clki->curr_freq == clki->min_freq) 5349 continue; 5350 ret = clk_set_rate(clki->clk, clki->min_freq); 5351 if (ret) { 5352 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n", 5353 __func__, clki->name, 5354 clki->min_freq, ret); 5355 break; 5356 } 5357 clki->curr_freq = clki->min_freq; 5358 } 5359 } 5360 dev_dbg(hba->dev, "%s: clk: %s, rate: %lu\n", __func__, 5361 clki->name, clk_get_rate(clki->clk)); 5362 } 5363 if (hba->vops->clk_scale_notify) 5364 hba->vops->clk_scale_notify(hba); 5365out: 5366 return ret; 5367} 5368 5369static int ufshcd_devfreq_target(struct device *dev, 5370 unsigned long *freq, u32 flags) 5371{ 5372 int err = 0; 5373 struct ufs_hba *hba = dev_get_drvdata(dev); 5374 5375 if (!ufshcd_is_clkscaling_enabled(hba)) 5376 return -EINVAL; 5377 5378 if (*freq == UINT_MAX) 5379 err = ufshcd_scale_clks(hba, true); 5380 else if (*freq == 0) 5381 err = ufshcd_scale_clks(hba, false); 5382 5383 return err; 5384} 5385 5386static int ufshcd_devfreq_get_dev_status(struct device *dev, 5387 struct devfreq_dev_status *stat) 5388{ 5389 struct ufs_hba *hba = dev_get_drvdata(dev); 5390 struct ufs_clk_scaling *scaling = &hba->clk_scaling; 5391 unsigned long flags; 5392 5393 if (!ufshcd_is_clkscaling_enabled(hba)) 5394 return -EINVAL; 5395 5396 memset(stat, 0, sizeof(*stat)); 5397 5398 spin_lock_irqsave(hba->host->host_lock, flags); 5399 if (!scaling->window_start_t) 5400 goto start_window; 5401 5402 if (scaling->is_busy_started) 5403 scaling->tot_busy_t += ktime_to_us(ktime_sub(ktime_get(), 5404 scaling->busy_start_t)); 5405 5406 stat->total_time = jiffies_to_usecs((long)jiffies - 5407 (long)scaling->window_start_t); 5408 stat->busy_time = scaling->tot_busy_t; 5409start_window: 5410 scaling->window_start_t = jiffies; 5411 scaling->tot_busy_t = 0; 5412 5413 if (hba->outstanding_reqs) { 5414 scaling->busy_start_t = ktime_get(); 5415 scaling->is_busy_started = true; 5416 } else { 5417 scaling->busy_start_t = ktime_set(0, 0); 5418 scaling->is_busy_started = false; 5419 } 5420 spin_unlock_irqrestore(hba->host->host_lock, flags); 5421 return 0; 5422} 5423 5424static struct devfreq_dev_profile ufs_devfreq_profile = { 5425 .polling_ms = 100, 5426 .target = ufshcd_devfreq_target, 5427 .get_dev_status = ufshcd_devfreq_get_dev_status, 5428}; 5429 5430/** 5431 * ufshcd_init - Driver initialization routine 5432 * @hba: per-adapter instance 5433 * @mmio_base: base register address 5434 * @irq: Interrupt line of device 5435 * Returns 0 on success, non-zero value on failure 5436 */ 5437int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) 5438{ 5439 int err; 5440 struct Scsi_Host *host = hba->host; 5441 struct device *dev = hba->dev; 5442 5443 if (!mmio_base) { 5444 dev_err(hba->dev, 5445 "Invalid memory reference for mmio_base is NULL\n"); 5446 err = -ENODEV; 5447 goto out_error; 5448 } 5449 5450 hba->mmio_base = mmio_base; 5451 hba->irq = irq; 5452 5453 err = ufshcd_hba_init(hba); 5454 if (err) 5455 goto out_error; 5456 5457 /* Read capabilities registers */ 5458 ufshcd_hba_capabilities(hba); 5459 5460 /* Get UFS version supported by the controller */ 5461 hba->ufs_version = ufshcd_get_ufs_version(hba); 5462 5463 /* Get Interrupt bit mask per version */ 5464 hba->intr_mask = ufshcd_get_intr_mask(hba); 5465 5466 err = ufshcd_set_dma_mask(hba); 5467 if (err) { 5468 dev_err(hba->dev, "set dma mask failed\n"); 5469 goto out_disable; 5470 } 5471 5472 /* Allocate memory for host memory space */ 5473 err = ufshcd_memory_alloc(hba); 5474 if (err) { 5475 dev_err(hba->dev, "Memory allocation failed\n"); 5476 goto out_disable; 5477 } 5478 5479 /* Configure LRB */ 5480 ufshcd_host_memory_configure(hba); 5481 5482 host->can_queue = hba->nutrs; 5483 host->cmd_per_lun = hba->nutrs; 5484 host->max_id = UFSHCD_MAX_ID; 5485 host->max_lun = UFS_MAX_LUNS; 5486 host->max_channel = UFSHCD_MAX_CHANNEL; 5487 host->unique_id = host->host_no; 5488 host->max_cmd_len = MAX_CDB_SIZE; 5489 5490 hba->max_pwr_info.is_valid = false; 5491 5492 /* Initailize wait queue for task management */ 5493 init_waitqueue_head(&hba->tm_wq); 5494 init_waitqueue_head(&hba->tm_tag_wq); 5495 5496 /* Initialize work queues */ 5497 INIT_WORK(&hba->eh_work, ufshcd_err_handler); 5498 INIT_WORK(&hba->eeh_work, ufshcd_exception_event_handler); 5499 5500 /* Initialize UIC command mutex */ 5501 mutex_init(&hba->uic_cmd_mutex); 5502 5503 /* Initialize mutex for device management commands */ 5504 mutex_init(&hba->dev_cmd.lock); 5505 5506 /* Initialize device management tag acquire wait queue */ 5507 init_waitqueue_head(&hba->dev_cmd.tag_wq); 5508 5509 ufshcd_init_clk_gating(hba); 5510 /* IRQ registration */ 5511 err = devm_request_irq(dev, irq, ufshcd_intr, IRQF_SHARED, UFSHCD, hba); 5512 if (err) { 5513 dev_err(hba->dev, "request irq failed\n"); 5514 goto exit_gating; 5515 } else { 5516 hba->is_irq_enabled = true; 5517 } 5518 5519 /* Enable SCSI tag mapping */ 5520 err = scsi_init_shared_tag_map(host, host->can_queue); 5521 if (err) { 5522 dev_err(hba->dev, "init shared queue failed\n"); 5523 goto exit_gating; 5524 } 5525 5526 err = scsi_add_host(host, hba->dev); 5527 if (err) { 5528 dev_err(hba->dev, "scsi_add_host failed\n"); 5529 goto exit_gating; 5530 } 5531 5532 /* Host controller enable */ 5533 err = ufshcd_hba_enable(hba); 5534 if (err) { 5535 dev_err(hba->dev, "Host controller enable failed\n"); 5536 goto out_remove_scsi_host; 5537 } 5538 5539 if (ufshcd_is_clkscaling_enabled(hba)) { 5540 hba->devfreq = devfreq_add_device(dev, &ufs_devfreq_profile, 5541 "simple_ondemand", NULL); 5542 if (IS_ERR(hba->devfreq)) { 5543 dev_err(hba->dev, "Unable to register with devfreq %ld\n", 5544 PTR_ERR(hba->devfreq)); 5545 goto out_remove_scsi_host; 5546 } 5547 /* Suspend devfreq until the UFS device is detected */ 5548 devfreq_suspend_device(hba->devfreq); 5549 hba->clk_scaling.window_start_t = 0; 5550 } 5551 5552 /* Hold auto suspend until async scan completes */ 5553 pm_runtime_get_sync(dev); 5554 5555 /* 5556 * The device-initialize-sequence hasn't been invoked yet. 5557 * Set the device to power-off state 5558 */ 5559 ufshcd_set_ufs_dev_poweroff(hba); 5560 5561 async_schedule(ufshcd_async_scan, hba); 5562 5563 return 0; 5564 5565out_remove_scsi_host: 5566 scsi_remove_host(hba->host); 5567exit_gating: 5568 ufshcd_exit_clk_gating(hba); 5569out_disable: 5570 hba->is_irq_enabled = false; 5571 scsi_host_put(host); 5572 ufshcd_hba_exit(hba); 5573out_error: 5574 return err; 5575} 5576EXPORT_SYMBOL_GPL(ufshcd_init); 5577 5578MODULE_AUTHOR("Santosh Yaragnavi <santosh.sy@samsung.com>"); 5579MODULE_AUTHOR("Vinayak Holikatti <h.vinayak@samsung.com>"); 5580MODULE_DESCRIPTION("Generic UFS host controller driver Core"); 5581MODULE_LICENSE("GPL"); 5582MODULE_VERSION(UFSHCD_DRIVER_VERSION); 5583