/* * Universal Flash Storage Host controller driver Core * * This code is based on drivers/scsi/ufs/ufshcd.c * Copyright (C) 2011-2013 Samsung India Software Operations * * Authors: * Santosh Yaraganavi * Vinayak Holikatti * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * See the COPYING file in the top-level directory or visit * * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * This program is provided "AS IS" and "WITH ALL FAULTS" and * without warranty of any kind. You are solely responsible for * determining the appropriateness of using and distributing * the program and assume all risks associated with your exercise * of rights with respect to the program, including but not limited * to infringement of third party rights, the risks and costs of * program errors, damage to or loss of data, programs or equipment, * and unavailability or interruption of operations. Under no * circumstances will the contributor of this Program be liable for * any damages of any kind arising from your use or distribution of * this program. */ #include #include "ufshcd.h" #include "unipro.h" #define UFSHCD_ENABLE_INTRS (UTP_TRANSFER_REQ_COMPL |\ UTP_TASK_REQ_COMPL |\ UIC_POWER_MODE |\ UFSHCD_ERROR_MASK) /* UIC command timeout, unit: ms */ #define UIC_CMD_TIMEOUT 500 /* NOP OUT retries waiting for NOP IN response */ #define NOP_OUT_RETRIES 10 /* Timeout after 30 msecs if NOP OUT hangs without response */ #define NOP_OUT_TIMEOUT 30 /* msecs */ /* Query request retries */ #define QUERY_REQ_RETRIES 10 /* Query request timeout */ #define QUERY_REQ_TIMEOUT 30 /* msec */ /* Task management command timeout */ #define TM_CMD_TIMEOUT 100 /* msecs */ /* Expose the flag value from utp_upiu_query.value */ #define MASK_QUERY_UPIU_FLAG_LOC 0xFF /* Interrupt aggregation default timeout, unit: 40us */ #define INT_AGGR_DEF_TO 0x02 enum { UFSHCD_MAX_CHANNEL = 0, UFSHCD_MAX_ID = 1, UFSHCD_MAX_LUNS = 8, UFSHCD_CMD_PER_LUN = 32, UFSHCD_CAN_QUEUE = 32, }; /* UFSHCD states */ enum { UFSHCD_STATE_RESET, UFSHCD_STATE_ERROR, UFSHCD_STATE_OPERATIONAL, }; /* UFSHCD error handling flags */ enum { UFSHCD_EH_IN_PROGRESS = (1 << 0), }; /* UFSHCD UIC layer error flags */ enum { UFSHCD_UIC_DL_PA_INIT_ERROR = (1 << 0), /* Data link layer error */ UFSHCD_UIC_NL_ERROR = (1 << 1), /* Network layer error */ UFSHCD_UIC_TL_ERROR = (1 << 2), /* Transport Layer error */ UFSHCD_UIC_DME_ERROR = (1 << 3), /* DME error */ }; /* Interrupt configuration options */ enum { UFSHCD_INT_DISABLE, UFSHCD_INT_ENABLE, UFSHCD_INT_CLEAR, }; #define ufshcd_set_eh_in_progress(h) \ (h->eh_flags |= UFSHCD_EH_IN_PROGRESS) #define ufshcd_eh_in_progress(h) \ (h->eh_flags & UFSHCD_EH_IN_PROGRESS) #define ufshcd_clear_eh_in_progress(h) \ (h->eh_flags &= ~UFSHCD_EH_IN_PROGRESS) static void ufshcd_tmc_handler(struct ufs_hba *hba); static void ufshcd_async_scan(void *data, async_cookie_t cookie); static int ufshcd_reset_and_restore(struct ufs_hba *hba); static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag); static int ufshcd_read_sdev_qdepth(struct ufs_hba *hba, struct scsi_device *sdev); /* * ufshcd_wait_for_register - wait for register value to change * @hba - per-adapter interface * @reg - mmio register offset * @mask - mask to apply to read register value * @val - wait condition * @interval_us - polling interval in microsecs * @timeout_ms - timeout in millisecs * * Returns -ETIMEDOUT on error, zero on success */ static int ufshcd_wait_for_register(struct ufs_hba *hba, u32 reg, u32 mask, u32 val, unsigned long interval_us, unsigned long timeout_ms) { int err = 0; unsigned long timeout = jiffies + msecs_to_jiffies(timeout_ms); /* ignore bits that we don't intend to wait on */ val = val & mask; while ((ufshcd_readl(hba, reg) & mask) != val) { /* wakeup within 50us of expiry */ usleep_range(interval_us, interval_us + 50); if (time_after(jiffies, timeout)) { if ((ufshcd_readl(hba, reg) & mask) != val) err = -ETIMEDOUT; break; } } return err; } /** * ufshcd_get_intr_mask - Get the interrupt bit mask * @hba - Pointer to adapter instance * * Returns interrupt bit mask per version */ static inline u32 ufshcd_get_intr_mask(struct ufs_hba *hba) { if (hba->ufs_version == UFSHCI_VERSION_10) return INTERRUPT_MASK_ALL_VER_10; else return INTERRUPT_MASK_ALL_VER_11; } /** * ufshcd_get_ufs_version - Get the UFS version supported by the HBA * @hba - Pointer to adapter instance * * Returns UFSHCI version supported by the controller */ static inline u32 ufshcd_get_ufs_version(struct ufs_hba *hba) { return ufshcd_readl(hba, REG_UFS_VERSION); } /** * ufshcd_is_device_present - Check if any device connected to * the host controller * @reg_hcs - host controller status register value * * Returns 1 if device present, 0 if no device detected */ static inline int ufshcd_is_device_present(u32 reg_hcs) { return (DEVICE_PRESENT & reg_hcs) ? 1 : 0; } /** * ufshcd_get_tr_ocs - Get the UTRD Overall Command Status * @lrb: pointer to local command reference block * * This function is used to get the OCS field from UTRD * Returns the OCS field in the UTRD */ static inline int ufshcd_get_tr_ocs(struct ufshcd_lrb *lrbp) { return le32_to_cpu(lrbp->utr_descriptor_ptr->header.dword_2) & MASK_OCS; } /** * ufshcd_get_tmr_ocs - Get the UTMRD Overall Command Status * @task_req_descp: pointer to utp_task_req_desc structure * * This function is used to get the OCS field from UTMRD * Returns the OCS field in the UTMRD */ static inline int ufshcd_get_tmr_ocs(struct utp_task_req_desc *task_req_descp) { return le32_to_cpu(task_req_descp->header.dword_2) & MASK_OCS; } /** * ufshcd_get_tm_free_slot - get a free slot for task management request * @hba: per adapter instance * @free_slot: pointer to variable with available slot value * * Get a free tag and lock it until ufshcd_put_tm_slot() is called. * Returns 0 if free slot is not available, else return 1 with tag value * in @free_slot. */ static bool ufshcd_get_tm_free_slot(struct ufs_hba *hba, int *free_slot) { int tag; bool ret = false; if (!free_slot) goto out; do { tag = find_first_zero_bit(&hba->tm_slots_in_use, hba->nutmrs); if (tag >= hba->nutmrs) goto out; } while (test_and_set_bit_lock(tag, &hba->tm_slots_in_use)); *free_slot = tag; ret = true; out: return ret; } static inline void ufshcd_put_tm_slot(struct ufs_hba *hba, int slot) { clear_bit_unlock(slot, &hba->tm_slots_in_use); } /** * ufshcd_utrl_clear - Clear a bit in UTRLCLR register * @hba: per adapter instance * @pos: position of the bit to be cleared */ static inline void ufshcd_utrl_clear(struct ufs_hba *hba, u32 pos) { ufshcd_writel(hba, ~(1 << pos), REG_UTP_TRANSFER_REQ_LIST_CLEAR); } /** * ufshcd_get_lists_status - Check UCRDY, UTRLRDY and UTMRLRDY * @reg: Register value of host controller status * * Returns integer, 0 on Success and positive value if failed */ static inline int ufshcd_get_lists_status(u32 reg) { /* * The mask 0xFF is for the following HCS register bits * Bit Description * 0 Device Present * 1 UTRLRDY * 2 UTMRLRDY * 3 UCRDY * 4 HEI * 5 DEI * 6-7 reserved */ return (((reg) & (0xFF)) >> 1) ^ (0x07); } /** * ufshcd_get_uic_cmd_result - Get the UIC command result * @hba: Pointer to adapter instance * * This function gets the result of UIC command completion * Returns 0 on success, non zero value on error */ static inline int ufshcd_get_uic_cmd_result(struct ufs_hba *hba) { return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2) & MASK_UIC_COMMAND_RESULT; } /** * ufshcd_get_dme_attr_val - Get the value of attribute returned by UIC command * @hba: Pointer to adapter instance * * This function gets UIC command argument3 * Returns 0 on success, non zero value on error */ static inline u32 ufshcd_get_dme_attr_val(struct ufs_hba *hba) { return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3); } /** * ufshcd_get_req_rsp - returns the TR response transaction type * @ucd_rsp_ptr: pointer to response UPIU */ static inline int ufshcd_get_req_rsp(struct utp_upiu_rsp *ucd_rsp_ptr) { return be32_to_cpu(ucd_rsp_ptr->header.dword_0) >> 24; } /** * ufshcd_get_rsp_upiu_result - Get the result from response UPIU * @ucd_rsp_ptr: pointer to response UPIU * * This function gets the response status and scsi_status from response UPIU * Returns the response result code. */ static inline int ufshcd_get_rsp_upiu_result(struct utp_upiu_rsp *ucd_rsp_ptr) { return be32_to_cpu(ucd_rsp_ptr->header.dword_1) & MASK_RSP_UPIU_RESULT; } /* * ufshcd_get_rsp_upiu_data_seg_len - Get the data segment length * from response UPIU * @ucd_rsp_ptr: pointer to response UPIU * * Return the data segment length. */ static inline unsigned int ufshcd_get_rsp_upiu_data_seg_len(struct utp_upiu_rsp *ucd_rsp_ptr) { return be32_to_cpu(ucd_rsp_ptr->header.dword_2) & MASK_RSP_UPIU_DATA_SEG_LEN; } /** * ufshcd_is_exception_event - Check if the device raised an exception event * @ucd_rsp_ptr: pointer to response UPIU * * The function checks if the device raised an exception event indicated in * the Device Information field of response UPIU. * * Returns true if exception is raised, false otherwise. */ static inline bool ufshcd_is_exception_event(struct utp_upiu_rsp *ucd_rsp_ptr) { return be32_to_cpu(ucd_rsp_ptr->header.dword_2) & MASK_RSP_EXCEPTION_EVENT ? true : false; } /** * ufshcd_reset_intr_aggr - Reset interrupt aggregation values. * @hba: per adapter instance */ static inline void ufshcd_reset_intr_aggr(struct ufs_hba *hba) { ufshcd_writel(hba, INT_AGGR_ENABLE | INT_AGGR_COUNTER_AND_TIMER_RESET, REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL); } /** * ufshcd_config_intr_aggr - Configure interrupt aggregation values. * @hba: per adapter instance * @cnt: Interrupt aggregation counter threshold * @tmout: Interrupt aggregation timeout value */ static inline void ufshcd_config_intr_aggr(struct ufs_hba *hba, u8 cnt, u8 tmout) { ufshcd_writel(hba, INT_AGGR_ENABLE | INT_AGGR_PARAM_WRITE | INT_AGGR_COUNTER_THLD_VAL(cnt) | INT_AGGR_TIMEOUT_VAL(tmout), REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL); } /** * ufshcd_enable_run_stop_reg - Enable run-stop registers, * When run-stop registers are set to 1, it indicates the * host controller that it can process the requests * @hba: per adapter instance */ static void ufshcd_enable_run_stop_reg(struct ufs_hba *hba) { ufshcd_writel(hba, UTP_TASK_REQ_LIST_RUN_STOP_BIT, REG_UTP_TASK_REQ_LIST_RUN_STOP); ufshcd_writel(hba, UTP_TRANSFER_REQ_LIST_RUN_STOP_BIT, REG_UTP_TRANSFER_REQ_LIST_RUN_STOP); } /** * ufshcd_hba_start - Start controller initialization sequence * @hba: per adapter instance */ static inline void ufshcd_hba_start(struct ufs_hba *hba) { ufshcd_writel(hba, CONTROLLER_ENABLE, REG_CONTROLLER_ENABLE); } /** * ufshcd_is_hba_active - Get controller state * @hba: per adapter instance * * Returns zero if controller is active, 1 otherwise */ static inline int ufshcd_is_hba_active(struct ufs_hba *hba) { return (ufshcd_readl(hba, REG_CONTROLLER_ENABLE) & 0x1) ? 0 : 1; } /** * ufshcd_send_command - Send SCSI or device management commands * @hba: per adapter instance * @task_tag: Task tag of the command */ static inline void ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag) { __set_bit(task_tag, &hba->outstanding_reqs); ufshcd_writel(hba, 1 << task_tag, REG_UTP_TRANSFER_REQ_DOOR_BELL); } /** * ufshcd_copy_sense_data - Copy sense data in case of check condition * @lrb - pointer to local reference block */ static inline void ufshcd_copy_sense_data(struct ufshcd_lrb *lrbp) { int len; if (lrbp->sense_buffer && ufshcd_get_rsp_upiu_data_seg_len(lrbp->ucd_rsp_ptr)) { len = be16_to_cpu(lrbp->ucd_rsp_ptr->sr.sense_data_len); memcpy(lrbp->sense_buffer, lrbp->ucd_rsp_ptr->sr.sense_data, min_t(int, len, SCSI_SENSE_BUFFERSIZE)); } } /** * ufshcd_copy_query_response() - Copy the Query Response and the data * descriptor * @hba: per adapter instance * @lrb - pointer to local reference block */ static int ufshcd_copy_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) { struct ufs_query_res *query_res = &hba->dev_cmd.query.response; memcpy(&query_res->upiu_res, &lrbp->ucd_rsp_ptr->qr, QUERY_OSF_SIZE); /* Get the descriptor */ if (lrbp->ucd_rsp_ptr->qr.opcode == UPIU_QUERY_OPCODE_READ_DESC) { u8 *descp = (u8 *)lrbp->ucd_rsp_ptr + GENERAL_UPIU_REQUEST_SIZE; u16 resp_len; u16 buf_len; /* data segment length */ resp_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) & MASK_QUERY_DATA_SEG_LEN; buf_len = hba->dev_cmd.query.request.upiu_req.length; if (likely(buf_len >= resp_len)) { memcpy(hba->dev_cmd.query.descriptor, descp, resp_len); } else { dev_warn(hba->dev, "%s: Response size is bigger than buffer", __func__); return -EINVAL; } } return 0; } /** * ufshcd_hba_capabilities - Read controller capabilities * @hba: per adapter instance */ static inline void ufshcd_hba_capabilities(struct ufs_hba *hba) { hba->capabilities = ufshcd_readl(hba, REG_CONTROLLER_CAPABILITIES); /* nutrs and nutmrs are 0 based values */ hba->nutrs = (hba->capabilities & MASK_TRANSFER_REQUESTS_SLOTS) + 1; hba->nutmrs = ((hba->capabilities & MASK_TASK_MANAGEMENT_REQUEST_SLOTS) >> 16) + 1; } /** * ufshcd_ready_for_uic_cmd - Check if controller is ready * to accept UIC commands * @hba: per adapter instance * Return true on success, else false */ static inline bool ufshcd_ready_for_uic_cmd(struct ufs_hba *hba) { if (ufshcd_readl(hba, REG_CONTROLLER_STATUS) & UIC_COMMAND_READY) return true; else return false; } /** * ufshcd_get_upmcrs - Get the power mode change request status * @hba: Pointer to adapter instance * * This function gets the UPMCRS field of HCS register * Returns value of UPMCRS field */ static inline u8 ufshcd_get_upmcrs(struct ufs_hba *hba) { return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) >> 8) & 0x7; } /** * ufshcd_dispatch_uic_cmd - Dispatch UIC commands to unipro layers * @hba: per adapter instance * @uic_cmd: UIC command * * Mutex must be held. */ static inline void ufshcd_dispatch_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd) { WARN_ON(hba->active_uic_cmd); hba->active_uic_cmd = uic_cmd; /* Write Args */ ufshcd_writel(hba, uic_cmd->argument1, REG_UIC_COMMAND_ARG_1); ufshcd_writel(hba, uic_cmd->argument2, REG_UIC_COMMAND_ARG_2); ufshcd_writel(hba, uic_cmd->argument3, REG_UIC_COMMAND_ARG_3); /* Write UIC Cmd */ ufshcd_writel(hba, uic_cmd->command & COMMAND_OPCODE_MASK, REG_UIC_COMMAND); } /** * ufshcd_wait_for_uic_cmd - Wait complectioin of UIC command * @hba: per adapter instance * @uic_command: UIC command * * Must be called with mutex held. * Returns 0 only if success. */ static int ufshcd_wait_for_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd) { int ret; unsigned long flags; if (wait_for_completion_timeout(&uic_cmd->done, msecs_to_jiffies(UIC_CMD_TIMEOUT))) ret = uic_cmd->argument2 & MASK_UIC_COMMAND_RESULT; else ret = -ETIMEDOUT; spin_lock_irqsave(hba->host->host_lock, flags); hba->active_uic_cmd = NULL; spin_unlock_irqrestore(hba->host->host_lock, flags); return ret; } /** * __ufshcd_send_uic_cmd - Send UIC commands and retrieve the result * @hba: per adapter instance * @uic_cmd: UIC command * * Identical to ufshcd_send_uic_cmd() expect mutex. Must be called * with mutex held. * Returns 0 only if success. */ static int __ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd) { int ret; unsigned long flags; if (!ufshcd_ready_for_uic_cmd(hba)) { dev_err(hba->dev, "Controller not ready to accept UIC commands\n"); return -EIO; } init_completion(&uic_cmd->done); spin_lock_irqsave(hba->host->host_lock, flags); ufshcd_dispatch_uic_cmd(hba, uic_cmd); spin_unlock_irqrestore(hba->host->host_lock, flags); ret = ufshcd_wait_for_uic_cmd(hba, uic_cmd); return ret; } /** * ufshcd_send_uic_cmd - Send UIC commands and retrieve the result * @hba: per adapter instance * @uic_cmd: UIC command * * Returns 0 only if success. */ static int ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd) { int ret; mutex_lock(&hba->uic_cmd_mutex); ret = __ufshcd_send_uic_cmd(hba, uic_cmd); mutex_unlock(&hba->uic_cmd_mutex); return ret; } /** * ufshcd_map_sg - Map scatter-gather list to prdt * @lrbp - pointer to local reference block * * Returns 0 in case of success, non-zero value in case of failure */ static int ufshcd_map_sg(struct ufshcd_lrb *lrbp) { struct ufshcd_sg_entry *prd_table; struct scatterlist *sg; struct scsi_cmnd *cmd; int sg_segments; int i; cmd = lrbp->cmd; sg_segments = scsi_dma_map(cmd); if (sg_segments < 0) return sg_segments; if (sg_segments) { lrbp->utr_descriptor_ptr->prd_table_length = cpu_to_le16((u16) (sg_segments)); prd_table = (struct ufshcd_sg_entry *)lrbp->ucd_prdt_ptr; scsi_for_each_sg(cmd, sg, sg_segments, i) { prd_table[i].size = cpu_to_le32(((u32) sg_dma_len(sg))-1); prd_table[i].base_addr = cpu_to_le32(lower_32_bits(sg->dma_address)); prd_table[i].upper_addr = cpu_to_le32(upper_32_bits(sg->dma_address)); } } else { lrbp->utr_descriptor_ptr->prd_table_length = 0; } return 0; } /** * ufshcd_enable_intr - enable interrupts * @hba: per adapter instance * @intrs: interrupt bits */ static void ufshcd_enable_intr(struct ufs_hba *hba, u32 intrs) { u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE); if (hba->ufs_version == UFSHCI_VERSION_10) { u32 rw; rw = set & INTERRUPT_MASK_RW_VER_10; set = rw | ((set ^ intrs) & intrs); } else { set |= intrs; } ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE); } /** * ufshcd_disable_intr - disable interrupts * @hba: per adapter instance * @intrs: interrupt bits */ static void ufshcd_disable_intr(struct ufs_hba *hba, u32 intrs) { u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE); if (hba->ufs_version == UFSHCI_VERSION_10) { u32 rw; rw = (set & INTERRUPT_MASK_RW_VER_10) & ~(intrs & INTERRUPT_MASK_RW_VER_10); set = rw | ((set & intrs) & ~INTERRUPT_MASK_RW_VER_10); } else { set &= ~intrs; } ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE); } /** * ufshcd_prepare_req_desc_hdr() - Fills the requests header * descriptor according to request * @lrbp: pointer to local reference block * @upiu_flags: flags required in the header * @cmd_dir: requests data direction */ static void ufshcd_prepare_req_desc_hdr(struct ufshcd_lrb *lrbp, u32 *upiu_flags, enum dma_data_direction cmd_dir) { struct utp_transfer_req_desc *req_desc = lrbp->utr_descriptor_ptr; u32 data_direction; u32 dword_0; if (cmd_dir == DMA_FROM_DEVICE) { data_direction = UTP_DEVICE_TO_HOST; *upiu_flags = UPIU_CMD_FLAGS_READ; } else if (cmd_dir == DMA_TO_DEVICE) { data_direction = UTP_HOST_TO_DEVICE; *upiu_flags = UPIU_CMD_FLAGS_WRITE; } else { data_direction = UTP_NO_DATA_TRANSFER; *upiu_flags = UPIU_CMD_FLAGS_NONE; } dword_0 = data_direction | (lrbp->command_type << UPIU_COMMAND_TYPE_OFFSET); if (lrbp->intr_cmd) dword_0 |= UTP_REQ_DESC_INT_CMD; /* Transfer request descriptor header fields */ req_desc->header.dword_0 = cpu_to_le32(dword_0); /* * assigning invalid value for command status. Controller * updates OCS on command completion, with the command * status */ req_desc->header.dword_2 = cpu_to_le32(OCS_INVALID_COMMAND_STATUS); } /** * ufshcd_prepare_utp_scsi_cmd_upiu() - fills the utp_transfer_req_desc, * for scsi commands * @lrbp - local reference block pointer * @upiu_flags - flags */ static void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb *lrbp, u32 upiu_flags) { struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr; /* command descriptor fields */ ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD( UPIU_TRANSACTION_COMMAND, upiu_flags, lrbp->lun, lrbp->task_tag); ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD( UPIU_COMMAND_SET_TYPE_SCSI, 0, 0, 0); /* Total EHS length and Data segment length will be zero */ ucd_req_ptr->header.dword_2 = 0; ucd_req_ptr->sc.exp_data_transfer_len = cpu_to_be32(lrbp->cmd->sdb.length); memcpy(ucd_req_ptr->sc.cdb, lrbp->cmd->cmnd, (min_t(unsigned short, lrbp->cmd->cmd_len, MAX_CDB_SIZE))); } /** * ufshcd_prepare_utp_query_req_upiu() - fills the utp_transfer_req_desc, * for query requsts * @hba: UFS hba * @lrbp: local reference block pointer * @upiu_flags: flags */ static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, u32 upiu_flags) { struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr; struct ufs_query *query = &hba->dev_cmd.query; u16 len = be16_to_cpu(query->request.upiu_req.length); u8 *descp = (u8 *)lrbp->ucd_req_ptr + GENERAL_UPIU_REQUEST_SIZE; /* Query request header */ ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD( UPIU_TRANSACTION_QUERY_REQ, upiu_flags, lrbp->lun, lrbp->task_tag); ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD( 0, query->request.query_func, 0, 0); /* Data segment length */ ucd_req_ptr->header.dword_2 = UPIU_HEADER_DWORD( 0, 0, len >> 8, (u8)len); /* Copy the Query Request buffer as is */ memcpy(&ucd_req_ptr->qr, &query->request.upiu_req, QUERY_OSF_SIZE); /* Copy the Descriptor */ if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC) memcpy(descp, query->descriptor, len); } static inline void ufshcd_prepare_utp_nop_upiu(struct ufshcd_lrb *lrbp) { struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr; memset(ucd_req_ptr, 0, sizeof(struct utp_upiu_req)); /* command descriptor fields */ ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD( UPIU_TRANSACTION_NOP_OUT, 0, 0, lrbp->task_tag); } /** * ufshcd_compose_upiu - form UFS Protocol Information Unit(UPIU) * @hba - per adapter instance * @lrb - pointer to local reference block */ static int ufshcd_compose_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) { u32 upiu_flags; int ret = 0; switch (lrbp->command_type) { case UTP_CMD_TYPE_SCSI: if (likely(lrbp->cmd)) { ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, lrbp->cmd->sc_data_direction); ufshcd_prepare_utp_scsi_cmd_upiu(lrbp, upiu_flags); } else { ret = -EINVAL; } break; case UTP_CMD_TYPE_DEV_MANAGE: ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, DMA_NONE); if (hba->dev_cmd.type == DEV_CMD_TYPE_QUERY) ufshcd_prepare_utp_query_req_upiu( hba, lrbp, upiu_flags); else if (hba->dev_cmd.type == DEV_CMD_TYPE_NOP) ufshcd_prepare_utp_nop_upiu(lrbp); else ret = -EINVAL; break; case UTP_CMD_TYPE_UFS: /* For UFS native command implementation */ ret = -ENOTSUPP; dev_err(hba->dev, "%s: UFS native command are not supported\n", __func__); break; default: ret = -ENOTSUPP; dev_err(hba->dev, "%s: unknown command type: 0x%x\n", __func__, lrbp->command_type); break; } /* end of switch */ return ret; } /** * ufshcd_queuecommand - main entry point for SCSI requests * @cmd: command from SCSI Midlayer * @done: call back function * * Returns 0 for success, non-zero in case of failure */ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd) { struct ufshcd_lrb *lrbp; struct ufs_hba *hba; unsigned long flags; int tag; int err = 0; hba = shost_priv(host); tag = cmd->request->tag; spin_lock_irqsave(hba->host->host_lock, flags); switch (hba->ufshcd_state) { case UFSHCD_STATE_OPERATIONAL: break; case UFSHCD_STATE_RESET: err = SCSI_MLQUEUE_HOST_BUSY; goto out_unlock; case UFSHCD_STATE_ERROR: set_host_byte(cmd, DID_ERROR); cmd->scsi_done(cmd); goto out_unlock; default: dev_WARN_ONCE(hba->dev, 1, "%s: invalid state %d\n", __func__, hba->ufshcd_state); set_host_byte(cmd, DID_BAD_TARGET); cmd->scsi_done(cmd); goto out_unlock; } spin_unlock_irqrestore(hba->host->host_lock, flags); /* acquire the tag to make sure device cmds don't use it */ if (test_and_set_bit_lock(tag, &hba->lrb_in_use)) { /* * Dev manage command in progress, requeue the command. * Requeuing the command helps in cases where the request *may* * find different tag instead of waiting for dev manage command * completion. */ err = SCSI_MLQUEUE_HOST_BUSY; goto out; } lrbp = &hba->lrb[tag]; WARN_ON(lrbp->cmd); lrbp->cmd = cmd; lrbp->sense_bufflen = SCSI_SENSE_BUFFERSIZE; lrbp->sense_buffer = cmd->sense_buffer; lrbp->task_tag = tag; lrbp->lun = cmd->device->lun; lrbp->intr_cmd = false; lrbp->command_type = UTP_CMD_TYPE_SCSI; /* form UPIU before issuing the command */ ufshcd_compose_upiu(hba, lrbp); err = ufshcd_map_sg(lrbp); if (err) { lrbp->cmd = NULL; clear_bit_unlock(tag, &hba->lrb_in_use); goto out; } /* issue command to the controller */ spin_lock_irqsave(hba->host->host_lock, flags); ufshcd_send_command(hba, tag); out_unlock: spin_unlock_irqrestore(hba->host->host_lock, flags); out: return err; } static int ufshcd_compose_dev_cmd(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, enum dev_cmd_type cmd_type, int tag) { lrbp->cmd = NULL; lrbp->sense_bufflen = 0; lrbp->sense_buffer = NULL; lrbp->task_tag = tag; lrbp->lun = 0; /* device management cmd is not specific to any LUN */ lrbp->command_type = UTP_CMD_TYPE_DEV_MANAGE; lrbp->intr_cmd = true; /* No interrupt aggregation */ hba->dev_cmd.type = cmd_type; return ufshcd_compose_upiu(hba, lrbp); } static int ufshcd_clear_cmd(struct ufs_hba *hba, int tag) { int err = 0; unsigned long flags; u32 mask = 1 << tag; /* clear outstanding transaction before retry */ spin_lock_irqsave(hba->host->host_lock, flags); ufshcd_utrl_clear(hba, tag); spin_unlock_irqrestore(hba->host->host_lock, flags); /* * wait for for h/w to clear corresponding bit in door-bell. * max. wait is 1 sec. */ err = ufshcd_wait_for_register(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL, mask, ~mask, 1000, 1000); return err; } static int ufshcd_check_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) { struct ufs_query_res *query_res = &hba->dev_cmd.query.response; /* Get the UPIU response */ query_res->response = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr) >> UPIU_RSP_CODE_OFFSET; return query_res->response; } /** * ufshcd_dev_cmd_completion() - handles device management command responses * @hba: per adapter instance * @lrbp: pointer to local reference block */ static int ufshcd_dev_cmd_completion(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) { int resp; int err = 0; resp = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr); switch (resp) { case UPIU_TRANSACTION_NOP_IN: if (hba->dev_cmd.type != DEV_CMD_TYPE_NOP) { err = -EINVAL; dev_err(hba->dev, "%s: unexpected response %x\n", __func__, resp); } break; case UPIU_TRANSACTION_QUERY_RSP: err = ufshcd_check_query_response(hba, lrbp); if (!err) err = ufshcd_copy_query_response(hba, lrbp); break; case UPIU_TRANSACTION_REJECT_UPIU: /* TODO: handle Reject UPIU Response */ err = -EPERM; dev_err(hba->dev, "%s: Reject UPIU not fully implemented\n", __func__); break; default: err = -EINVAL; dev_err(hba->dev, "%s: Invalid device management cmd response: %x\n", __func__, resp); break; } return err; } static int ufshcd_wait_for_dev_cmd(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, int max_timeout) { int err = 0; unsigned long time_left; unsigned long flags; time_left = wait_for_completion_timeout(hba->dev_cmd.complete, msecs_to_jiffies(max_timeout)); spin_lock_irqsave(hba->host->host_lock, flags); hba->dev_cmd.complete = NULL; if (likely(time_left)) { err = ufshcd_get_tr_ocs(lrbp); if (!err) err = ufshcd_dev_cmd_completion(hba, lrbp); } spin_unlock_irqrestore(hba->host->host_lock, flags); if (!time_left) { err = -ETIMEDOUT; if (!ufshcd_clear_cmd(hba, lrbp->task_tag)) /* sucessfully cleared the command, retry if needed */ err = -EAGAIN; } return err; } /** * ufshcd_get_dev_cmd_tag - Get device management command tag * @hba: per-adapter instance * @tag: pointer to variable with available slot value * * Get a free slot and lock it until device management command * completes. * * Returns false if free slot is unavailable for locking, else * return true with tag value in @tag. */ static bool ufshcd_get_dev_cmd_tag(struct ufs_hba *hba, int *tag_out) { int tag; bool ret = false; unsigned long tmp; if (!tag_out) goto out; do { tmp = ~hba->lrb_in_use; tag = find_last_bit(&tmp, hba->nutrs); if (tag >= hba->nutrs) goto out; } while (test_and_set_bit_lock(tag, &hba->lrb_in_use)); *tag_out = tag; ret = true; out: return ret; } static inline void ufshcd_put_dev_cmd_tag(struct ufs_hba *hba, int tag) { clear_bit_unlock(tag, &hba->lrb_in_use); } /** * ufshcd_exec_dev_cmd - API for sending device management requests * @hba - UFS hba * @cmd_type - specifies the type (NOP, Query...) * @timeout - time in seconds * * NOTE: Since there is only one available tag for device management commands, * it is expected you hold the hba->dev_cmd.lock mutex. */ static int ufshcd_exec_dev_cmd(struct ufs_hba *hba, enum dev_cmd_type cmd_type, int timeout) { struct ufshcd_lrb *lrbp; int err; int tag; struct completion wait; unsigned long flags; /* * Get free slot, sleep if slots are unavailable. * Even though we use wait_event() which sleeps indefinitely, * the maximum wait time is bounded by SCSI request timeout. */ wait_event(hba->dev_cmd.tag_wq, ufshcd_get_dev_cmd_tag(hba, &tag)); init_completion(&wait); lrbp = &hba->lrb[tag]; WARN_ON(lrbp->cmd); err = ufshcd_compose_dev_cmd(hba, lrbp, cmd_type, tag); if (unlikely(err)) goto out_put_tag; hba->dev_cmd.complete = &wait; spin_lock_irqsave(hba->host->host_lock, flags); ufshcd_send_command(hba, tag); spin_unlock_irqrestore(hba->host->host_lock, flags); err = ufshcd_wait_for_dev_cmd(hba, lrbp, timeout); out_put_tag: ufshcd_put_dev_cmd_tag(hba, tag); wake_up(&hba->dev_cmd.tag_wq); return err; } /** * ufshcd_init_query() - init the query response and request parameters * @hba: per-adapter instance * @request: address of the request pointer to be initialized * @response: address of the response pointer to be initialized * @opcode: operation to perform * @idn: flag idn to access * @index: LU number to access * @selector: query/flag/descriptor further identification */ static inline void ufshcd_init_query(struct ufs_hba *hba, struct ufs_query_req **request, struct ufs_query_res **response, enum query_opcode opcode, u8 idn, u8 index, u8 selector) { *request = &hba->dev_cmd.query.request; *response = &hba->dev_cmd.query.response; memset(*request, 0, sizeof(struct ufs_query_req)); memset(*response, 0, sizeof(struct ufs_query_res)); (*request)->upiu_req.opcode = opcode; (*request)->upiu_req.idn = idn; (*request)->upiu_req.index = index; (*request)->upiu_req.selector = selector; } /** * ufshcd_query_flag() - API function for sending flag query requests * hba: per-adapter instance * query_opcode: flag query to perform * idn: flag idn to access * flag_res: the flag value after the query request completes * * Returns 0 for success, non-zero in case of failure */ static int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode, enum flag_idn idn, bool *flag_res) { struct ufs_query_req *request = NULL; struct ufs_query_res *response = NULL; int err, index = 0, selector = 0; BUG_ON(!hba); mutex_lock(&hba->dev_cmd.lock); ufshcd_init_query(hba, &request, &response, opcode, idn, index, selector); switch (opcode) { case UPIU_QUERY_OPCODE_SET_FLAG: case UPIU_QUERY_OPCODE_CLEAR_FLAG: case UPIU_QUERY_OPCODE_TOGGLE_FLAG: request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST; break; case UPIU_QUERY_OPCODE_READ_FLAG: request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST; if (!flag_res) { /* No dummy reads */ dev_err(hba->dev, "%s: Invalid argument for read request\n", __func__); err = -EINVAL; goto out_unlock; } break; default: dev_err(hba->dev, "%s: Expected query flag opcode but got = %d\n", __func__, opcode); err = -EINVAL; goto out_unlock; } err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT); if (err) { dev_err(hba->dev, "%s: Sending flag query for idn %d failed, err = %d\n", __func__, idn, err); goto out_unlock; } if (flag_res) *flag_res = (be32_to_cpu(response->upiu_res.value) & MASK_QUERY_UPIU_FLAG_LOC) & 0x1; out_unlock: mutex_unlock(&hba->dev_cmd.lock); return err; } /** * ufshcd_query_attr - API function for sending attribute requests * hba: per-adapter instance * opcode: attribute opcode * idn: attribute idn to access * index: index field * selector: selector field * attr_val: the attribute value after the query request completes * * Returns 0 for success, non-zero in case of failure */ static int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode, enum attr_idn idn, u8 index, u8 selector, u32 *attr_val) { struct ufs_query_req *request = NULL; struct ufs_query_res *response = NULL; int err; BUG_ON(!hba); if (!attr_val) { dev_err(hba->dev, "%s: attribute value required for opcode 0x%x\n", __func__, opcode); err = -EINVAL; goto out; } mutex_lock(&hba->dev_cmd.lock); ufshcd_init_query(hba, &request, &response, opcode, idn, index, selector); switch (opcode) { case UPIU_QUERY_OPCODE_WRITE_ATTR: request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST; request->upiu_req.value = cpu_to_be32(*attr_val); break; case UPIU_QUERY_OPCODE_READ_ATTR: request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST; break; default: dev_err(hba->dev, "%s: Expected query attr opcode but got = 0x%.2x\n", __func__, opcode); err = -EINVAL; goto out_unlock; } err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT); if (err) { dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, err = %d\n", __func__, opcode, idn, err); goto out_unlock; } *attr_val = be32_to_cpu(response->upiu_res.value); out_unlock: mutex_unlock(&hba->dev_cmd.lock); out: return err; } /** * ufshcd_query_descriptor - API function for sending descriptor requests * hba: per-adapter instance * opcode: attribute opcode * idn: attribute idn to access * index: index field * selector: selector field * desc_buf: the buffer that contains the descriptor * buf_len: length parameter passed to the device * * Returns 0 for success, non-zero in case of failure. * The buf_len parameter will contain, on return, the length parameter * received on the response. */ int ufshcd_query_descriptor(struct ufs_hba *hba, enum query_opcode opcode, enum desc_idn idn, u8 index, u8 selector, u8 *desc_buf, int *buf_len) { struct ufs_query_req *request = NULL; struct ufs_query_res *response = NULL; int err; BUG_ON(!hba); if (!desc_buf) { dev_err(hba->dev, "%s: descriptor buffer required for opcode 0x%x\n", __func__, opcode); err = -EINVAL; goto out; } if (*buf_len <= QUERY_DESC_MIN_SIZE || *buf_len > QUERY_DESC_MAX_SIZE) { dev_err(hba->dev, "%s: descriptor buffer size (%d) is out of range\n", __func__, *buf_len); err = -EINVAL; goto out; } mutex_lock(&hba->dev_cmd.lock); ufshcd_init_query(hba, &request, &response, opcode, idn, index, selector); hba->dev_cmd.query.descriptor = desc_buf; request->upiu_req.length = *buf_len; switch (opcode) { case UPIU_QUERY_OPCODE_WRITE_DESC: request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST; break; case UPIU_QUERY_OPCODE_READ_DESC: request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST; break; default: dev_err(hba->dev, "%s: Expected query descriptor opcode but got = 0x%.2x\n", __func__, opcode); err = -EINVAL; goto out_unlock; } err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT); if (err) { dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, err = %d\n", __func__, opcode, idn, err); goto out_unlock; } hba->dev_cmd.query.descriptor = NULL; *buf_len = response->upiu_res.length; out_unlock: mutex_unlock(&hba->dev_cmd.lock); out: return err; } /** * ufshcd_memory_alloc - allocate memory for host memory space data structures * @hba: per adapter instance * * 1. Allocate DMA memory for Command Descriptor array * Each command descriptor consist of Command UPIU, Response UPIU and PRDT * 2. Allocate DMA memory for UTP Transfer Request Descriptor List (UTRDL). * 3. Allocate DMA memory for UTP Task Management Request Descriptor List * (UTMRDL) * 4. Allocate memory for local reference block(lrb). * * Returns 0 for success, non-zero in case of failure */ static int ufshcd_memory_alloc(struct ufs_hba *hba) { size_t utmrdl_size, utrdl_size, ucdl_size; /* Allocate memory for UTP command descriptors */ ucdl_size = (sizeof(struct utp_transfer_cmd_desc) * hba->nutrs); hba->ucdl_base_addr = dmam_alloc_coherent(hba->dev, ucdl_size, &hba->ucdl_dma_addr, GFP_KERNEL); /* * UFSHCI requires UTP command descriptor to be 128 byte aligned. * make sure hba->ucdl_dma_addr is aligned to PAGE_SIZE * if hba->ucdl_dma_addr is aligned to PAGE_SIZE, then it will * be aligned to 128 bytes as well */ if (!hba->ucdl_base_addr || WARN_ON(hba->ucdl_dma_addr & (PAGE_SIZE - 1))) { dev_err(hba->dev, "Command Descriptor Memory allocation failed\n"); goto out; } /* * Allocate memory for UTP Transfer descriptors * UFSHCI requires 1024 byte alignment of UTRD */ utrdl_size = (sizeof(struct utp_transfer_req_desc) * hba->nutrs); hba->utrdl_base_addr = dmam_alloc_coherent(hba->dev, utrdl_size, &hba->utrdl_dma_addr, GFP_KERNEL); if (!hba->utrdl_base_addr || WARN_ON(hba->utrdl_dma_addr & (PAGE_SIZE - 1))) { dev_err(hba->dev, "Transfer Descriptor Memory allocation failed\n"); goto out; } /* * Allocate memory for UTP Task Management descriptors * UFSHCI requires 1024 byte alignment of UTMRD */ utmrdl_size = sizeof(struct utp_task_req_desc) * hba->nutmrs; hba->utmrdl_base_addr = dmam_alloc_coherent(hba->dev, utmrdl_size, &hba->utmrdl_dma_addr, GFP_KERNEL); if (!hba->utmrdl_base_addr || WARN_ON(hba->utmrdl_dma_addr & (PAGE_SIZE - 1))) { dev_err(hba->dev, "Task Management Descriptor Memory allocation failed\n"); goto out; } /* Allocate memory for local reference block */ hba->lrb = devm_kzalloc(hba->dev, hba->nutrs * sizeof(struct ufshcd_lrb), GFP_KERNEL); if (!hba->lrb) { dev_err(hba->dev, "LRB Memory allocation failed\n"); goto out; } return 0; out: return -ENOMEM; } /** * ufshcd_host_memory_configure - configure local reference block with * memory offsets * @hba: per adapter instance * * Configure Host memory space * 1. Update Corresponding UTRD.UCDBA and UTRD.UCDBAU with UCD DMA * address. * 2. Update each UTRD with Response UPIU offset, Response UPIU length * and PRDT offset. * 3. Save the corresponding addresses of UTRD, UCD.CMD, UCD.RSP and UCD.PRDT * into local reference block. */ static void ufshcd_host_memory_configure(struct ufs_hba *hba) { struct utp_transfer_cmd_desc *cmd_descp; struct utp_transfer_req_desc *utrdlp; dma_addr_t cmd_desc_dma_addr; dma_addr_t cmd_desc_element_addr; u16 response_offset; u16 prdt_offset; int cmd_desc_size; int i; utrdlp = hba->utrdl_base_addr; cmd_descp = hba->ucdl_base_addr; response_offset = offsetof(struct utp_transfer_cmd_desc, response_upiu); prdt_offset = offsetof(struct utp_transfer_cmd_desc, prd_table); cmd_desc_size = sizeof(struct utp_transfer_cmd_desc); cmd_desc_dma_addr = hba->ucdl_dma_addr; for (i = 0; i < hba->nutrs; i++) { /* Configure UTRD with command descriptor base address */ cmd_desc_element_addr = (cmd_desc_dma_addr + (cmd_desc_size * i)); utrdlp[i].command_desc_base_addr_lo = cpu_to_le32(lower_32_bits(cmd_desc_element_addr)); utrdlp[i].command_desc_base_addr_hi = cpu_to_le32(upper_32_bits(cmd_desc_element_addr)); /* Response upiu and prdt offset should be in double words */ utrdlp[i].response_upiu_offset = cpu_to_le16((response_offset >> 2)); utrdlp[i].prd_table_offset = cpu_to_le16((prdt_offset >> 2)); utrdlp[i].response_upiu_length = cpu_to_le16(ALIGNED_UPIU_SIZE >> 2); hba->lrb[i].utr_descriptor_ptr = (utrdlp + i); hba->lrb[i].ucd_req_ptr = (struct utp_upiu_req *)(cmd_descp + i); hba->lrb[i].ucd_rsp_ptr = (struct utp_upiu_rsp *)cmd_descp[i].response_upiu; hba->lrb[i].ucd_prdt_ptr = (struct ufshcd_sg_entry *)cmd_descp[i].prd_table; } } /** * ufshcd_dme_link_startup - Notify Unipro to perform link startup * @hba: per adapter instance * * UIC_CMD_DME_LINK_STARTUP command must be issued to Unipro layer, * in order to initialize the Unipro link startup procedure. * Once the Unipro links are up, the device connected to the controller * is detected. * * Returns 0 on success, non-zero value on failure */ static int ufshcd_dme_link_startup(struct ufs_hba *hba) { struct uic_command uic_cmd = {0}; int ret; uic_cmd.command = UIC_CMD_DME_LINK_STARTUP; ret = ufshcd_send_uic_cmd(hba, &uic_cmd); if (ret) dev_err(hba->dev, "dme-link-startup: error code %d\n", ret); return ret; } /** * ufshcd_dme_set_attr - UIC command for DME_SET, DME_PEER_SET * @hba: per adapter instance * @attr_sel: uic command argument1 * @attr_set: attribute set type as uic command argument2 * @mib_val: setting value as uic command argument3 * @peer: indicate whether peer or local * * Returns 0 on success, non-zero value on failure */ int ufshcd_dme_set_attr(struct ufs_hba *hba, u32 attr_sel, u8 attr_set, u32 mib_val, u8 peer) { struct uic_command uic_cmd = {0}; static const char *const action[] = { "dme-set", "dme-peer-set" }; const char *set = action[!!peer]; int ret; uic_cmd.command = peer ? UIC_CMD_DME_PEER_SET : UIC_CMD_DME_SET; uic_cmd.argument1 = attr_sel; uic_cmd.argument2 = UIC_ARG_ATTR_TYPE(attr_set); uic_cmd.argument3 = mib_val; ret = ufshcd_send_uic_cmd(hba, &uic_cmd); if (ret) dev_err(hba->dev, "%s: attr-id 0x%x val 0x%x error code %d\n", set, UIC_GET_ATTR_ID(attr_sel), mib_val, ret); return ret; } EXPORT_SYMBOL_GPL(ufshcd_dme_set_attr); /** * ufshcd_dme_get_attr - UIC command for DME_GET, DME_PEER_GET * @hba: per adapter instance * @attr_sel: uic command argument1 * @mib_val: the value of the attribute as returned by the UIC command * @peer: indicate whether peer or local * * Returns 0 on success, non-zero value on failure */ int ufshcd_dme_get_attr(struct ufs_hba *hba, u32 attr_sel, u32 *mib_val, u8 peer) { struct uic_command uic_cmd = {0}; static const char *const action[] = { "dme-get", "dme-peer-get" }; const char *get = action[!!peer]; int ret; uic_cmd.command = peer ? UIC_CMD_DME_PEER_GET : UIC_CMD_DME_GET; uic_cmd.argument1 = attr_sel; ret = ufshcd_send_uic_cmd(hba, &uic_cmd); if (ret) { dev_err(hba->dev, "%s: attr-id 0x%x error code %d\n", get, UIC_GET_ATTR_ID(attr_sel), ret); goto out; } if (mib_val) *mib_val = uic_cmd.argument3; out: return ret; } EXPORT_SYMBOL_GPL(ufshcd_dme_get_attr); /** * ufshcd_uic_change_pwr_mode - Perform the UIC power mode chage * using DME_SET primitives. * @hba: per adapter instance * @mode: powr mode value * * Returns 0 on success, non-zero value on failure */ static int ufshcd_uic_change_pwr_mode(struct ufs_hba *hba, u8 mode) { struct uic_command uic_cmd = {0}; struct completion pwr_done; unsigned long flags; u8 status; int ret; uic_cmd.command = UIC_CMD_DME_SET; uic_cmd.argument1 = UIC_ARG_MIB(PA_PWRMODE); uic_cmd.argument3 = mode; init_completion(&pwr_done); mutex_lock(&hba->uic_cmd_mutex); spin_lock_irqsave(hba->host->host_lock, flags); hba->pwr_done = &pwr_done; spin_unlock_irqrestore(hba->host->host_lock, flags); ret = __ufshcd_send_uic_cmd(hba, &uic_cmd); if (ret) { dev_err(hba->dev, "pwr mode change with mode 0x%x uic error %d\n", mode, ret); goto out; } if (!wait_for_completion_timeout(hba->pwr_done, msecs_to_jiffies(UIC_CMD_TIMEOUT))) { dev_err(hba->dev, "pwr mode change with mode 0x%x completion timeout\n", mode); ret = -ETIMEDOUT; goto out; } status = ufshcd_get_upmcrs(hba); if (status != PWR_LOCAL) { dev_err(hba->dev, "pwr mode change failed, host umpcrs:0x%x\n", status); ret = (status != PWR_OK) ? status : -1; } out: spin_lock_irqsave(hba->host->host_lock, flags); hba->pwr_done = NULL; spin_unlock_irqrestore(hba->host->host_lock, flags); mutex_unlock(&hba->uic_cmd_mutex); return ret; } /** * ufshcd_config_max_pwr_mode - Set & Change power mode with * maximum capability attribute information. * @hba: per adapter instance * * Returns 0 on success, non-zero value on failure */ static int ufshcd_config_max_pwr_mode(struct ufs_hba *hba) { enum {RX = 0, TX = 1}; u32 lanes[] = {1, 1}; u32 gear[] = {1, 1}; u8 pwr[] = {FASTAUTO_MODE, FASTAUTO_MODE}; int ret; /* Get the connected lane count */ ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDRXDATALANES), &lanes[RX]); ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES), &lanes[TX]); /* * First, get the maximum gears of HS speed. * If a zero value, it means there is no HSGEAR capability. * Then, get the maximum gears of PWM speed. */ ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR), &gear[RX]); if (!gear[RX]) { ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR), &gear[RX]); pwr[RX] = SLOWAUTO_MODE; } ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR), &gear[TX]); if (!gear[TX]) { ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR), &gear[TX]); pwr[TX] = SLOWAUTO_MODE; } /* * Configure attributes for power mode change with below. * - PA_RXGEAR, PA_ACTIVERXDATALANES, PA_RXTERMINATION, * - PA_TXGEAR, PA_ACTIVETXDATALANES, PA_TXTERMINATION, * - PA_HSSERIES */ ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXGEAR), gear[RX]); ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVERXDATALANES), lanes[RX]); if (pwr[RX] == FASTAUTO_MODE) ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), TRUE); ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXGEAR), gear[TX]); ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVETXDATALANES), lanes[TX]); if (pwr[TX] == FASTAUTO_MODE) ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), TRUE); if (pwr[RX] == FASTAUTO_MODE || pwr[TX] == FASTAUTO_MODE) ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HSSERIES), PA_HS_MODE_B); ret = ufshcd_uic_change_pwr_mode(hba, pwr[RX] << 4 | pwr[TX]); if (ret) dev_err(hba->dev, "pwr_mode: power mode change failed %d\n", ret); return ret; } /** * ufshcd_complete_dev_init() - checks device readiness * hba: per-adapter instance * * Set fDeviceInit flag and poll until device toggles it. */ static int ufshcd_complete_dev_init(struct ufs_hba *hba) { int i, retries, err = 0; bool flag_res = 1; for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) { /* Set the fDeviceInit flag */ err = ufshcd_query_flag(hba, UPIU_QUERY_OPCODE_SET_FLAG, QUERY_FLAG_IDN_FDEVICEINIT, NULL); if (!err || err == -ETIMEDOUT) break; dev_dbg(hba->dev, "%s: error %d retrying\n", __func__, err); } if (err) { dev_err(hba->dev, "%s setting fDeviceInit flag failed with error %d\n", __func__, err); goto out; } /* poll for max. 100 iterations for fDeviceInit flag to clear */ for (i = 0; i < 100 && !err && flag_res; i++) { for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) { err = ufshcd_query_flag(hba, UPIU_QUERY_OPCODE_READ_FLAG, QUERY_FLAG_IDN_FDEVICEINIT, &flag_res); if (!err || err == -ETIMEDOUT) break; dev_dbg(hba->dev, "%s: error %d retrying\n", __func__, err); } } if (err) dev_err(hba->dev, "%s reading fDeviceInit flag failed with error %d\n", __func__, err); else if (flag_res) dev_err(hba->dev, "%s fDeviceInit was not cleared by the device\n", __func__); out: return err; } /** * ufshcd_make_hba_operational - Make UFS controller operational * @hba: per adapter instance * * To bring UFS host controller to operational state, * 1. Check if device is present * 2. Enable required interrupts * 3. Configure interrupt aggregation * 4. Program UTRL and UTMRL base addres * 5. Configure run-stop-registers * * Returns 0 on success, non-zero value on failure */ static int ufshcd_make_hba_operational(struct ufs_hba *hba) { int err = 0; u32 reg; /* check if device present */ reg = ufshcd_readl(hba, REG_CONTROLLER_STATUS); if (!ufshcd_is_device_present(reg)) { dev_err(hba->dev, "cc: Device not present\n"); err = -ENXIO; goto out; } /* Enable required interrupts */ ufshcd_enable_intr(hba, UFSHCD_ENABLE_INTRS); /* Configure interrupt aggregation */ ufshcd_config_intr_aggr(hba, hba->nutrs - 1, INT_AGGR_DEF_TO); /* Configure UTRL and UTMRL base address registers */ ufshcd_writel(hba, lower_32_bits(hba->utrdl_dma_addr), REG_UTP_TRANSFER_REQ_LIST_BASE_L); ufshcd_writel(hba, upper_32_bits(hba->utrdl_dma_addr), REG_UTP_TRANSFER_REQ_LIST_BASE_H); ufshcd_writel(hba, lower_32_bits(hba->utmrdl_dma_addr), REG_UTP_TASK_REQ_LIST_BASE_L); ufshcd_writel(hba, upper_32_bits(hba->utmrdl_dma_addr), REG_UTP_TASK_REQ_LIST_BASE_H); /* * UCRDY, UTMRLDY and UTRLRDY bits must be 1 * DEI, HEI bits must be 0 */ if (!(ufshcd_get_lists_status(reg))) { ufshcd_enable_run_stop_reg(hba); } else { dev_err(hba->dev, "Host controller not ready to process requests"); err = -EIO; goto out; } out: return err; } /** * ufshcd_hba_enable - initialize the controller * @hba: per adapter instance * * The controller resets itself and controller firmware initialization * sequence kicks off. When controller is ready it will set * the Host Controller Enable bit to 1. * * Returns 0 on success, non-zero value on failure */ static int ufshcd_hba_enable(struct ufs_hba *hba) { int retry; /* * msleep of 1 and 5 used in this function might result in msleep(20), * but it was necessary to send the UFS FPGA to reset mode during * development and testing of this driver. msleep can be changed to * mdelay and retry count can be reduced based on the controller. */ if (!ufshcd_is_hba_active(hba)) { /* change controller state to "reset state" */ ufshcd_hba_stop(hba); /* * This delay is based on the testing done with UFS host * controller FPGA. The delay can be changed based on the * host controller used. */ msleep(5); } /* start controller initialization sequence */ ufshcd_hba_start(hba); /* * To initialize a UFS host controller HCE bit must be set to 1. * During initialization the HCE bit value changes from 1->0->1. * When the host controller completes initialization sequence * it sets the value of HCE bit to 1. The same HCE bit is read back * to check if the controller has completed initialization sequence. * So without this delay the value HCE = 1, set in the previous * instruction might be read back. * This delay can be changed based on the controller. */ msleep(1); /* wait for the host controller to complete initialization */ retry = 10; while (ufshcd_is_hba_active(hba)) { if (retry) { retry--; } else { dev_err(hba->dev, "Controller enable failed\n"); return -EIO; } msleep(5); } return 0; } /** * ufshcd_link_startup - Initialize unipro link startup * @hba: per adapter instance * * Returns 0 for success, non-zero in case of failure */ static int ufshcd_link_startup(struct ufs_hba *hba) { int ret; /* enable UIC related interrupts */ ufshcd_enable_intr(hba, UIC_COMMAND_COMPL); ret = ufshcd_dme_link_startup(hba); if (ret) goto out; ret = ufshcd_make_hba_operational(hba); out: if (ret) dev_err(hba->dev, "link startup failed %d\n", ret); return ret; } /** * ufshcd_verify_dev_init() - Verify device initialization * @hba: per-adapter instance * * Send NOP OUT UPIU and wait for NOP IN response to check whether the * device Transport Protocol (UTP) layer is ready after a reset. * If the UTP layer at the device side is not initialized, it may * not respond with NOP IN UPIU within timeout of %NOP_OUT_TIMEOUT * and we retry sending NOP OUT for %NOP_OUT_RETRIES iterations. */ static int ufshcd_verify_dev_init(struct ufs_hba *hba) { int err = 0; int retries; mutex_lock(&hba->dev_cmd.lock); for (retries = NOP_OUT_RETRIES; retries > 0; retries--) { err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_NOP, NOP_OUT_TIMEOUT); if (!err || err == -ETIMEDOUT) break; dev_dbg(hba->dev, "%s: error %d retrying\n", __func__, err); } mutex_unlock(&hba->dev_cmd.lock); if (err) dev_err(hba->dev, "%s: NOP OUT failed %d\n", __func__, err); return err; } /** * ufshcd_slave_alloc - handle initial SCSI device configurations * @sdev: pointer to SCSI device * * Returns success */ static int ufshcd_slave_alloc(struct scsi_device *sdev) { struct ufs_hba *hba; int lun_qdepth; hba = shost_priv(sdev->host); sdev->tagged_supported = 1; /* Mode sense(6) is not supported by UFS, so use Mode sense(10) */ sdev->use_10_for_ms = 1; scsi_set_tag_type(sdev, MSG_SIMPLE_TAG); /* allow SCSI layer to restart the device in case of errors */ sdev->allow_restart = 1; /* REPORT SUPPORTED OPERATION CODES is not supported */ sdev->no_report_opcodes = 1; lun_qdepth = ufshcd_read_sdev_qdepth(hba, sdev); if (lun_qdepth <= 0) /* eventually, we can figure out the real queue depth */ lun_qdepth = hba->nutrs; else lun_qdepth = min_t(int, lun_qdepth, hba->nutrs); dev_dbg(hba->dev, "%s: activate tcq with queue depth %d\n", __func__, lun_qdepth); scsi_activate_tcq(sdev, lun_qdepth); return 0; } /** * ufshcd_change_queue_depth - change queue depth * @sdev: pointer to SCSI device * @depth: required depth to set * @reason: reason for changing the depth * * Change queue depth according to the reason and make sure * the max. limits are not crossed. */ int ufshcd_change_queue_depth(struct scsi_device *sdev, int depth, int reason) { struct ufs_hba *hba = shost_priv(sdev->host); if (depth > hba->nutrs) depth = hba->nutrs; switch (reason) { case SCSI_QDEPTH_DEFAULT: case SCSI_QDEPTH_RAMP_UP: if (!sdev->tagged_supported) depth = 1; scsi_adjust_queue_depth(sdev, scsi_get_tag_type(sdev), depth); break; case SCSI_QDEPTH_QFULL: scsi_track_queue_full(sdev, depth); break; default: return -EOPNOTSUPP; } return depth; } /** * ufshcd_slave_configure - adjust SCSI device configurations * @sdev: pointer to SCSI device */ static int ufshcd_slave_configure(struct scsi_device *sdev) { struct request_queue *q = sdev->request_queue; blk_queue_update_dma_pad(q, PRDT_DATA_BYTE_COUNT_PAD - 1); blk_queue_max_segment_size(q, PRDT_DATA_BYTE_COUNT_MAX); return 0; } /** * ufshcd_slave_destroy - remove SCSI device configurations * @sdev: pointer to SCSI device */ static void ufshcd_slave_destroy(struct scsi_device *sdev) { struct ufs_hba *hba; hba = shost_priv(sdev->host); scsi_deactivate_tcq(sdev, hba->nutrs); } /** * ufshcd_task_req_compl - handle task management request completion * @hba: per adapter instance * @index: index of the completed request * @resp: task management service response * * Returns non-zero value on error, zero on success */ static int ufshcd_task_req_compl(struct ufs_hba *hba, u32 index, u8 *resp) { struct utp_task_req_desc *task_req_descp; struct utp_upiu_task_rsp *task_rsp_upiup; unsigned long flags; int ocs_value; int task_result; spin_lock_irqsave(hba->host->host_lock, flags); /* Clear completed tasks from outstanding_tasks */ __clear_bit(index, &hba->outstanding_tasks); task_req_descp = hba->utmrdl_base_addr; ocs_value = ufshcd_get_tmr_ocs(&task_req_descp[index]); if (ocs_value == OCS_SUCCESS) { task_rsp_upiup = (struct utp_upiu_task_rsp *) task_req_descp[index].task_rsp_upiu; task_result = be32_to_cpu(task_rsp_upiup->header.dword_1); task_result = ((task_result & MASK_TASK_RESPONSE) >> 8); if (resp) *resp = (u8)task_result; } else { dev_err(hba->dev, "%s: failed, ocs = 0x%x\n", __func__, ocs_value); } spin_unlock_irqrestore(hba->host->host_lock, flags); return ocs_value; } /** * ufshcd_scsi_cmd_status - Update SCSI command result based on SCSI status * @lrb: pointer to local reference block of completed command * @scsi_status: SCSI command status * * Returns value base on SCSI command status */ static inline int ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status) { int result = 0; switch (scsi_status) { case SAM_STAT_CHECK_CONDITION: ufshcd_copy_sense_data(lrbp); case SAM_STAT_GOOD: result |= DID_OK << 16 | COMMAND_COMPLETE << 8 | scsi_status; break; case SAM_STAT_TASK_SET_FULL: case SAM_STAT_BUSY: case SAM_STAT_TASK_ABORTED: ufshcd_copy_sense_data(lrbp); result |= scsi_status; break; default: result |= DID_ERROR << 16; break; } /* end of switch */ return result; } /** * ufshcd_transfer_rsp_status - Get overall status of the response * @hba: per adapter instance * @lrb: pointer to local reference block of completed command * * Returns result of the command to notify SCSI midlayer */ static inline int ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) { int result = 0; int scsi_status; int ocs; /* overall command status of utrd */ ocs = ufshcd_get_tr_ocs(lrbp); switch (ocs) { case OCS_SUCCESS: result = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr); switch (result) { case UPIU_TRANSACTION_RESPONSE: /* * get the response UPIU result to extract * the SCSI command status */ result = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr); /* * get the result based on SCSI status response * to notify the SCSI midlayer of the command status */ scsi_status = result & MASK_SCSI_STATUS; result = ufshcd_scsi_cmd_status(lrbp, scsi_status); if (ufshcd_is_exception_event(lrbp->ucd_rsp_ptr)) schedule_work(&hba->eeh_work); break; case UPIU_TRANSACTION_REJECT_UPIU: /* TODO: handle Reject UPIU Response */ result = DID_ERROR << 16; dev_err(hba->dev, "Reject UPIU not fully implemented\n"); break; default: result = DID_ERROR << 16; dev_err(hba->dev, "Unexpected request response code = %x\n", result); break; } break; case OCS_ABORTED: result |= DID_ABORT << 16; break; case OCS_INVALID_COMMAND_STATUS: result |= DID_REQUEUE << 16; break; case OCS_INVALID_CMD_TABLE_ATTR: case OCS_INVALID_PRDT_ATTR: case OCS_MISMATCH_DATA_BUF_SIZE: case OCS_MISMATCH_RESP_UPIU_SIZE: case OCS_PEER_COMM_FAILURE: case OCS_FATAL_ERROR: default: result |= DID_ERROR << 16; dev_err(hba->dev, "OCS error from controller = %x\n", ocs); break; } /* end of switch */ return result; } /** * ufshcd_uic_cmd_compl - handle completion of uic command * @hba: per adapter instance * @intr_status: interrupt status generated by the controller */ static void ufshcd_uic_cmd_compl(struct ufs_hba *hba, u32 intr_status) { if ((intr_status & UIC_COMMAND_COMPL) && hba->active_uic_cmd) { hba->active_uic_cmd->argument2 |= ufshcd_get_uic_cmd_result(hba); hba->active_uic_cmd->argument3 = ufshcd_get_dme_attr_val(hba); complete(&hba->active_uic_cmd->done); } if ((intr_status & UIC_POWER_MODE) && hba->pwr_done) complete(hba->pwr_done); } /** * ufshcd_transfer_req_compl - handle SCSI and query command completion * @hba: per adapter instance */ static void ufshcd_transfer_req_compl(struct ufs_hba *hba) { struct ufshcd_lrb *lrbp; struct scsi_cmnd *cmd; unsigned long completed_reqs; u32 tr_doorbell; int result; int index; /* Resetting interrupt aggregation counters first and reading the * DOOR_BELL afterward allows us to handle all the completed requests. * In order to prevent other interrupts starvation the DB is read once * after reset. The down side of this solution is the possibility of * false interrupt if device completes another request after resetting * aggregation and before reading the DB. */ ufshcd_reset_intr_aggr(hba); tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL); completed_reqs = tr_doorbell ^ hba->outstanding_reqs; for_each_set_bit(index, &completed_reqs, hba->nutrs) { lrbp = &hba->lrb[index]; cmd = lrbp->cmd; if (cmd) { result = ufshcd_transfer_rsp_status(hba, lrbp); scsi_dma_unmap(cmd); cmd->result = result; /* Mark completed command as NULL in LRB */ lrbp->cmd = NULL; clear_bit_unlock(index, &hba->lrb_in_use); /* Do not touch lrbp after scsi done */ cmd->scsi_done(cmd); } else if (lrbp->command_type == UTP_CMD_TYPE_DEV_MANAGE) { if (hba->dev_cmd.complete) complete(hba->dev_cmd.complete); } } /* clear corresponding bits of completed commands */ hba->outstanding_reqs ^= completed_reqs; /* we might have free'd some tags above */ wake_up(&hba->dev_cmd.tag_wq); } /** * ufshcd_disable_ee - disable exception event * @hba: per-adapter instance * @mask: exception event to disable * * Disables exception event in the device so that the EVENT_ALERT * bit is not set. * * Returns zero on success, non-zero error value on failure. */ static int ufshcd_disable_ee(struct ufs_hba *hba, u16 mask) { int err = 0; u32 val; if (!(hba->ee_ctrl_mask & mask)) goto out; val = hba->ee_ctrl_mask & ~mask; val &= 0xFFFF; /* 2 bytes */ err = ufshcd_query_attr(hba, UPIU_QUERY_OPCODE_WRITE_ATTR, QUERY_ATTR_IDN_EE_CONTROL, 0, 0, &val); if (!err) hba->ee_ctrl_mask &= ~mask; out: return err; } /** * ufshcd_enable_ee - enable exception event * @hba: per-adapter instance * @mask: exception event to enable * * Enable corresponding exception event in the device to allow * device to alert host in critical scenarios. * * Returns zero on success, non-zero error value on failure. */ static int ufshcd_enable_ee(struct ufs_hba *hba, u16 mask) { int err = 0; u32 val; if (hba->ee_ctrl_mask & mask) goto out; val = hba->ee_ctrl_mask | mask; val &= 0xFFFF; /* 2 bytes */ err = ufshcd_query_attr(hba, UPIU_QUERY_OPCODE_WRITE_ATTR, QUERY_ATTR_IDN_EE_CONTROL, 0, 0, &val); if (!err) hba->ee_ctrl_mask |= mask; out: return err; } /** * ufshcd_enable_auto_bkops - Allow device managed BKOPS * @hba: per-adapter instance * * Allow device to manage background operations on its own. Enabling * this might lead to inconsistent latencies during normal data transfers * as the device is allowed to manage its own way of handling background * operations. * * Returns zero on success, non-zero on failure. */ static int ufshcd_enable_auto_bkops(struct ufs_hba *hba) { int err = 0; if (hba->auto_bkops_enabled) goto out; err = ufshcd_query_flag(hba, UPIU_QUERY_OPCODE_SET_FLAG, QUERY_FLAG_IDN_BKOPS_EN, NULL); if (err) { dev_err(hba->dev, "%s: failed to enable bkops %d\n", __func__, err); goto out; } hba->auto_bkops_enabled = true; /* No need of URGENT_BKOPS exception from the device */ err = ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS); if (err) dev_err(hba->dev, "%s: failed to disable exception event %d\n", __func__, err); out: return err; } /** * ufshcd_disable_auto_bkops - block device in doing background operations * @hba: per-adapter instance * * Disabling background operations improves command response latency but * has drawback of device moving into critical state where the device is * not-operable. Make sure to call ufshcd_enable_auto_bkops() whenever the * host is idle so that BKOPS are managed effectively without any negative * impacts. * * Returns zero on success, non-zero on failure. */ static int ufshcd_disable_auto_bkops(struct ufs_hba *hba) { int err = 0; if (!hba->auto_bkops_enabled) goto out; /* * If host assisted BKOPs is to be enabled, make sure * urgent bkops exception is allowed. */ err = ufshcd_enable_ee(hba, MASK_EE_URGENT_BKOPS); if (err) { dev_err(hba->dev, "%s: failed to enable exception event %d\n", __func__, err); goto out; } err = ufshcd_query_flag(hba, UPIU_QUERY_OPCODE_CLEAR_FLAG, QUERY_FLAG_IDN_BKOPS_EN, NULL); if (err) { dev_err(hba->dev, "%s: failed to disable bkops %d\n", __func__, err); ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS); goto out; } hba->auto_bkops_enabled = false; out: return err; } /** * ufshcd_force_reset_auto_bkops - force enable of auto bkops * @hba: per adapter instance * * After a device reset the device may toggle the BKOPS_EN flag * to default value. The s/w tracking variables should be updated * as well. Do this by forcing enable of auto bkops. */ static void ufshcd_force_reset_auto_bkops(struct ufs_hba *hba) { hba->auto_bkops_enabled = false; hba->ee_ctrl_mask |= MASK_EE_URGENT_BKOPS; ufshcd_enable_auto_bkops(hba); } static inline int ufshcd_get_bkops_status(struct ufs_hba *hba, u32 *status) { return ufshcd_query_attr(hba, UPIU_QUERY_OPCODE_READ_ATTR, QUERY_ATTR_IDN_BKOPS_STATUS, 0, 0, status); } /** * ufshcd_urgent_bkops - handle urgent bkops exception event * @hba: per-adapter instance * * Enable fBackgroundOpsEn flag in the device to permit background * operations. */ static int ufshcd_urgent_bkops(struct ufs_hba *hba) { int err; u32 status = 0; err = ufshcd_get_bkops_status(hba, &status); if (err) { dev_err(hba->dev, "%s: failed to get BKOPS status %d\n", __func__, err); goto out; } status = status & 0xF; /* handle only if status indicates performance impact or critical */ if (status >= BKOPS_STATUS_PERF_IMPACT) err = ufshcd_enable_auto_bkops(hba); out: return err; } static inline int ufshcd_get_ee_status(struct ufs_hba *hba, u32 *status) { return ufshcd_query_attr(hba, UPIU_QUERY_OPCODE_READ_ATTR, QUERY_ATTR_IDN_EE_STATUS, 0, 0, status); } /** * ufshcd_exception_event_handler - handle exceptions raised by device * @work: pointer to work data * * Read bExceptionEventStatus attribute from the device and handle the * exception event accordingly. */ static void ufshcd_exception_event_handler(struct work_struct *work) { struct ufs_hba *hba; int err; u32 status = 0; hba = container_of(work, struct ufs_hba, eeh_work); pm_runtime_get_sync(hba->dev); err = ufshcd_get_ee_status(hba, &status); if (err) { dev_err(hba->dev, "%s: failed to get exception status %d\n", __func__, err); goto out; } status &= hba->ee_ctrl_mask; if (status & MASK_EE_URGENT_BKOPS) { err = ufshcd_urgent_bkops(hba); if (err) dev_err(hba->dev, "%s: failed to handle urgent bkops %d\n", __func__, err); } out: pm_runtime_put_sync(hba->dev); return; } /** * ufshcd_err_handler - handle UFS errors that require s/w attention * @work: pointer to work structure */ static void ufshcd_err_handler(struct work_struct *work) { struct ufs_hba *hba; unsigned long flags; u32 err_xfer = 0; u32 err_tm = 0; int err = 0; int tag; hba = container_of(work, struct ufs_hba, eh_work); pm_runtime_get_sync(hba->dev); spin_lock_irqsave(hba->host->host_lock, flags); if (hba->ufshcd_state == UFSHCD_STATE_RESET) { spin_unlock_irqrestore(hba->host->host_lock, flags); goto out; } hba->ufshcd_state = UFSHCD_STATE_RESET; ufshcd_set_eh_in_progress(hba); /* Complete requests that have door-bell cleared by h/w */ ufshcd_transfer_req_compl(hba); ufshcd_tmc_handler(hba); spin_unlock_irqrestore(hba->host->host_lock, flags); /* Clear pending transfer requests */ for_each_set_bit(tag, &hba->outstanding_reqs, hba->nutrs) if (ufshcd_clear_cmd(hba, tag)) err_xfer |= 1 << tag; /* Clear pending task management requests */ for_each_set_bit(tag, &hba->outstanding_tasks, hba->nutmrs) if (ufshcd_clear_tm_cmd(hba, tag)) err_tm |= 1 << tag; /* Complete the requests that are cleared by s/w */ spin_lock_irqsave(hba->host->host_lock, flags); ufshcd_transfer_req_compl(hba); ufshcd_tmc_handler(hba); spin_unlock_irqrestore(hba->host->host_lock, flags); /* Fatal errors need reset */ if (err_xfer || err_tm || (hba->saved_err & INT_FATAL_ERRORS) || ((hba->saved_err & UIC_ERROR) && (hba->saved_uic_err & UFSHCD_UIC_DL_PA_INIT_ERROR))) { err = ufshcd_reset_and_restore(hba); if (err) { dev_err(hba->dev, "%s: reset and restore failed\n", __func__); hba->ufshcd_state = UFSHCD_STATE_ERROR; } /* * Inform scsi mid-layer that we did reset and allow to handle * Unit Attention properly. */ scsi_report_bus_reset(hba->host, 0); hba->saved_err = 0; hba->saved_uic_err = 0; } ufshcd_clear_eh_in_progress(hba); out: scsi_unblock_requests(hba->host); pm_runtime_put_sync(hba->dev); } /** * ufshcd_update_uic_error - check and set fatal UIC error flags. * @hba: per-adapter instance */ static void ufshcd_update_uic_error(struct ufs_hba *hba) { u32 reg; /* PA_INIT_ERROR is fatal and needs UIC reset */ reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DATA_LINK_LAYER); if (reg & UIC_DATA_LINK_LAYER_ERROR_PA_INIT) hba->uic_error |= UFSHCD_UIC_DL_PA_INIT_ERROR; /* UIC NL/TL/DME errors needs software retry */ reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_NETWORK_LAYER); if (reg) hba->uic_error |= UFSHCD_UIC_NL_ERROR; reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_TRANSPORT_LAYER); if (reg) hba->uic_error |= UFSHCD_UIC_TL_ERROR; reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DME); if (reg) hba->uic_error |= UFSHCD_UIC_DME_ERROR; dev_dbg(hba->dev, "%s: UIC error flags = 0x%08x\n", __func__, hba->uic_error); } /** * ufshcd_check_errors - Check for errors that need s/w attention * @hba: per-adapter instance */ static void ufshcd_check_errors(struct ufs_hba *hba) { bool queue_eh_work = false; if (hba->errors & INT_FATAL_ERRORS) queue_eh_work = true; if (hba->errors & UIC_ERROR) { hba->uic_error = 0; ufshcd_update_uic_error(hba); if (hba->uic_error) queue_eh_work = true; } if (queue_eh_work) { /* handle fatal errors only when link is functional */ if (hba->ufshcd_state == UFSHCD_STATE_OPERATIONAL) { /* block commands from scsi mid-layer */ scsi_block_requests(hba->host); /* transfer error masks to sticky bits */ hba->saved_err |= hba->errors; hba->saved_uic_err |= hba->uic_error; hba->ufshcd_state = UFSHCD_STATE_ERROR; schedule_work(&hba->eh_work); } } /* * if (!queue_eh_work) - * Other errors are either non-fatal where host recovers * itself without s/w intervention or errors that will be * handled by the SCSI core layer. */ } /** * ufshcd_tmc_handler - handle task management function completion * @hba: per adapter instance */ static void ufshcd_tmc_handler(struct ufs_hba *hba) { u32 tm_doorbell; tm_doorbell = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL); hba->tm_condition = tm_doorbell ^ hba->outstanding_tasks; wake_up(&hba->tm_wq); } /** * ufshcd_sl_intr - Interrupt service routine * @hba: per adapter instance * @intr_status: contains interrupts generated by the controller */ static void ufshcd_sl_intr(struct ufs_hba *hba, u32 intr_status) { hba->errors = UFSHCD_ERROR_MASK & intr_status; if (hba->errors) ufshcd_check_errors(hba); if (intr_status & UFSHCD_UIC_MASK) ufshcd_uic_cmd_compl(hba, intr_status); if (intr_status & UTP_TASK_REQ_COMPL) ufshcd_tmc_handler(hba); if (intr_status & UTP_TRANSFER_REQ_COMPL) ufshcd_transfer_req_compl(hba); } /** * ufshcd_intr - Main interrupt service routine * @irq: irq number * @__hba: pointer to adapter instance * * Returns IRQ_HANDLED - If interrupt is valid * IRQ_NONE - If invalid interrupt */ static irqreturn_t ufshcd_intr(int irq, void *__hba) { u32 intr_status; irqreturn_t retval = IRQ_NONE; struct ufs_hba *hba = __hba; spin_lock(hba->host->host_lock); intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS); if (intr_status) { ufshcd_writel(hba, intr_status, REG_INTERRUPT_STATUS); ufshcd_sl_intr(hba, intr_status); retval = IRQ_HANDLED; } spin_unlock(hba->host->host_lock); return retval; } static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag) { int err = 0; u32 mask = 1 << tag; unsigned long flags; if (!test_bit(tag, &hba->outstanding_tasks)) goto out; spin_lock_irqsave(hba->host->host_lock, flags); ufshcd_writel(hba, ~(1 << tag), REG_UTP_TASK_REQ_LIST_CLEAR); spin_unlock_irqrestore(hba->host->host_lock, flags); /* poll for max. 1 sec to clear door bell register by h/w */ err = ufshcd_wait_for_register(hba, REG_UTP_TASK_REQ_DOOR_BELL, mask, 0, 1000, 1000); out: return err; } /** * ufshcd_issue_tm_cmd - issues task management commands to controller * @hba: per adapter instance * @lun_id: LUN ID to which TM command is sent * @task_id: task ID to which the TM command is applicable * @tm_function: task management function opcode * @tm_response: task management service response return value * * Returns non-zero value on error, zero on success. */ static int ufshcd_issue_tm_cmd(struct ufs_hba *hba, int lun_id, int task_id, u8 tm_function, u8 *tm_response) { struct utp_task_req_desc *task_req_descp; struct utp_upiu_task_req *task_req_upiup; struct Scsi_Host *host; unsigned long flags; int free_slot; int err; int task_tag; host = hba->host; /* * Get free slot, sleep if slots are unavailable. * Even though we use wait_event() which sleeps indefinitely, * the maximum wait time is bounded by %TM_CMD_TIMEOUT. */ wait_event(hba->tm_tag_wq, ufshcd_get_tm_free_slot(hba, &free_slot)); spin_lock_irqsave(host->host_lock, flags); task_req_descp = hba->utmrdl_base_addr; task_req_descp += free_slot; /* Configure task request descriptor */ task_req_descp->header.dword_0 = cpu_to_le32(UTP_REQ_DESC_INT_CMD); task_req_descp->header.dword_2 = cpu_to_le32(OCS_INVALID_COMMAND_STATUS); /* Configure task request UPIU */ task_req_upiup = (struct utp_upiu_task_req *) task_req_descp->task_req_upiu; task_tag = hba->nutrs + free_slot; task_req_upiup->header.dword_0 = UPIU_HEADER_DWORD(UPIU_TRANSACTION_TASK_REQ, 0, lun_id, task_tag); task_req_upiup->header.dword_1 = UPIU_HEADER_DWORD(0, tm_function, 0, 0); task_req_upiup->input_param1 = cpu_to_be32(lun_id); task_req_upiup->input_param2 = cpu_to_be32(task_id); /* send command to the controller */ __set_bit(free_slot, &hba->outstanding_tasks); ufshcd_writel(hba, 1 << free_slot, REG_UTP_TASK_REQ_DOOR_BELL); spin_unlock_irqrestore(host->host_lock, flags); /* wait until the task management command is completed */ err = wait_event_timeout(hba->tm_wq, test_bit(free_slot, &hba->tm_condition), msecs_to_jiffies(TM_CMD_TIMEOUT)); if (!err) { dev_err(hba->dev, "%s: task management cmd 0x%.2x timed-out\n", __func__, tm_function); if (ufshcd_clear_tm_cmd(hba, free_slot)) dev_WARN(hba->dev, "%s: unable clear tm cmd (slot %d) after timeout\n", __func__, free_slot); err = -ETIMEDOUT; } else { err = ufshcd_task_req_compl(hba, free_slot, tm_response); } clear_bit(free_slot, &hba->tm_condition); ufshcd_put_tm_slot(hba, free_slot); wake_up(&hba->tm_tag_wq); return err; } /** * ufshcd_eh_device_reset_handler - device reset handler registered to * scsi layer. * @cmd: SCSI command pointer * * Returns SUCCESS/FAILED */ static int ufshcd_eh_device_reset_handler(struct scsi_cmnd *cmd) { struct Scsi_Host *host; struct ufs_hba *hba; unsigned int tag; u32 pos; int err; u8 resp = 0xF; struct ufshcd_lrb *lrbp; unsigned long flags; host = cmd->device->host; hba = shost_priv(host); tag = cmd->request->tag; lrbp = &hba->lrb[tag]; err = ufshcd_issue_tm_cmd(hba, lrbp->lun, 0, UFS_LOGICAL_RESET, &resp); if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) { if (!err) err = resp; goto out; } /* clear the commands that were pending for corresponding LUN */ for_each_set_bit(pos, &hba->outstanding_reqs, hba->nutrs) { if (hba->lrb[pos].lun == lrbp->lun) { err = ufshcd_clear_cmd(hba, pos); if (err) break; } } spin_lock_irqsave(host->host_lock, flags); ufshcd_transfer_req_compl(hba); spin_unlock_irqrestore(host->host_lock, flags); out: if (!err) { err = SUCCESS; } else { dev_err(hba->dev, "%s: failed with err %d\n", __func__, err); err = FAILED; } return err; } /** * ufshcd_abort - abort a specific command * @cmd: SCSI command pointer * * Abort the pending command in device by sending UFS_ABORT_TASK task management * command, and in host controller by clearing the door-bell register. There can * be race between controller sending the command to the device while abort is * issued. To avoid that, first issue UFS_QUERY_TASK to check if the command is * really issued and then try to abort it. * * Returns SUCCESS/FAILED */ static int ufshcd_abort(struct scsi_cmnd *cmd) { struct Scsi_Host *host; struct ufs_hba *hba; unsigned long flags; unsigned int tag; int err = 0; int poll_cnt; u8 resp = 0xF; struct ufshcd_lrb *lrbp; u32 reg; host = cmd->device->host; hba = shost_priv(host); tag = cmd->request->tag; /* If command is already aborted/completed, return SUCCESS */ if (!(test_bit(tag, &hba->outstanding_reqs))) goto out; reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL); if (!(reg & (1 << tag))) { dev_err(hba->dev, "%s: cmd was completed, but without a notifying intr, tag = %d", __func__, tag); } lrbp = &hba->lrb[tag]; for (poll_cnt = 100; poll_cnt; poll_cnt--) { err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag, UFS_QUERY_TASK, &resp); if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_SUCCEEDED) { /* cmd pending in the device */ break; } else if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_COMPL) { /* * cmd not pending in the device, check if it is * in transition. */ reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL); if (reg & (1 << tag)) { /* sleep for max. 200us to stabilize */ usleep_range(100, 200); continue; } /* command completed already */ goto out; } else { if (!err) err = resp; /* service response error */ goto out; } } if (!poll_cnt) { err = -EBUSY; goto out; } err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag, UFS_ABORT_TASK, &resp); if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) { if (!err) err = resp; /* service response error */ goto out; } err = ufshcd_clear_cmd(hba, tag); if (err) goto out; scsi_dma_unmap(cmd); spin_lock_irqsave(host->host_lock, flags); __clear_bit(tag, &hba->outstanding_reqs); hba->lrb[tag].cmd = NULL; spin_unlock_irqrestore(host->host_lock, flags); clear_bit_unlock(tag, &hba->lrb_in_use); wake_up(&hba->dev_cmd.tag_wq); out: if (!err) { err = SUCCESS; } else { dev_err(hba->dev, "%s: failed with err %d\n", __func__, err); err = FAILED; } return err; } /** * ufshcd_host_reset_and_restore - reset and restore host controller * @hba: per-adapter instance * * Note that host controller reset may issue DME_RESET to * local and remote (device) Uni-Pro stack and the attributes * are reset to default state. * * Returns zero on success, non-zero on failure */ static int ufshcd_host_reset_and_restore(struct ufs_hba *hba) { int err; async_cookie_t cookie; unsigned long flags; /* Reset the host controller */ spin_lock_irqsave(hba->host->host_lock, flags); ufshcd_hba_stop(hba); spin_unlock_irqrestore(hba->host->host_lock, flags); err = ufshcd_hba_enable(hba); if (err) goto out; /* Establish the link again and restore the device */ cookie = async_schedule(ufshcd_async_scan, hba); /* wait for async scan to be completed */ async_synchronize_cookie(++cookie); if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL) err = -EIO; out: if (err) dev_err(hba->dev, "%s: Host init failed %d\n", __func__, err); return err; } /** * ufshcd_reset_and_restore - reset and re-initialize host/device * @hba: per-adapter instance * * Reset and recover device, host and re-establish link. This * is helpful to recover the communication in fatal error conditions. * * Returns zero on success, non-zero on failure */ static int ufshcd_reset_and_restore(struct ufs_hba *hba) { int err = 0; unsigned long flags; err = ufshcd_host_reset_and_restore(hba); /* * After reset the door-bell might be cleared, complete * outstanding requests in s/w here. */ spin_lock_irqsave(hba->host->host_lock, flags); ufshcd_transfer_req_compl(hba); ufshcd_tmc_handler(hba); spin_unlock_irqrestore(hba->host->host_lock, flags); return err; } /** * ufshcd_eh_host_reset_handler - host reset handler registered to scsi layer * @cmd - SCSI command pointer * * Returns SUCCESS/FAILED */ static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd) { int err; unsigned long flags; struct ufs_hba *hba; hba = shost_priv(cmd->device->host); /* * Check if there is any race with fatal error handling. * If so, wait for it to complete. Even though fatal error * handling does reset and restore in some cases, don't assume * anything out of it. We are just avoiding race here. */ do { spin_lock_irqsave(hba->host->host_lock, flags); if (!(work_pending(&hba->eh_work) || hba->ufshcd_state == UFSHCD_STATE_RESET)) break; spin_unlock_irqrestore(hba->host->host_lock, flags); dev_dbg(hba->dev, "%s: reset in progress\n", __func__); flush_work(&hba->eh_work); } while (1); hba->ufshcd_state = UFSHCD_STATE_RESET; ufshcd_set_eh_in_progress(hba); spin_unlock_irqrestore(hba->host->host_lock, flags); err = ufshcd_reset_and_restore(hba); spin_lock_irqsave(hba->host->host_lock, flags); if (!err) { err = SUCCESS; hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL; } else { err = FAILED; hba->ufshcd_state = UFSHCD_STATE_ERROR; } ufshcd_clear_eh_in_progress(hba); spin_unlock_irqrestore(hba->host->host_lock, flags); return err; } /** * ufshcd_read_sdev_qdepth - read the lun command queue depth * @hba: Pointer to adapter instance * @sdev: pointer to SCSI device * * Return in case of success the lun's queue depth else error. */ static int ufshcd_read_sdev_qdepth(struct ufs_hba *hba, struct scsi_device *sdev) { int ret; int buff_len = UNIT_DESC_MAX_SIZE; u8 desc_buf[UNIT_DESC_MAX_SIZE]; ret = ufshcd_query_descriptor(hba, UPIU_QUERY_OPCODE_READ_DESC, QUERY_DESC_IDN_UNIT, sdev->lun, 0, desc_buf, &buff_len); if (ret || (buff_len < UNIT_DESC_PARAM_LU_Q_DEPTH)) { dev_err(hba->dev, "%s:Failed reading unit descriptor. len = %d ret = %d" , __func__, buff_len, ret); if (!ret) ret = -EINVAL; goto out; } ret = desc_buf[UNIT_DESC_PARAM_LU_Q_DEPTH] & 0xFF; out: return ret; } /** * ufshcd_async_scan - asynchronous execution for link startup * @data: data pointer to pass to this function * @cookie: cookie data */ static void ufshcd_async_scan(void *data, async_cookie_t cookie) { struct ufs_hba *hba = (struct ufs_hba *)data; int ret; ret = ufshcd_link_startup(hba); if (ret) goto out; ufshcd_config_max_pwr_mode(hba); ret = ufshcd_verify_dev_init(hba); if (ret) goto out; ret = ufshcd_complete_dev_init(hba); if (ret) goto out; ufshcd_force_reset_auto_bkops(hba); hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL; /* If we are in error handling context no need to scan the host */ if (!ufshcd_eh_in_progress(hba)) { scsi_scan_host(hba->host); pm_runtime_put_sync(hba->dev); } out: return; } static struct scsi_host_template ufshcd_driver_template = { .module = THIS_MODULE, .name = UFSHCD, .proc_name = UFSHCD, .queuecommand = ufshcd_queuecommand, .slave_alloc = ufshcd_slave_alloc, .slave_configure = ufshcd_slave_configure, .slave_destroy = ufshcd_slave_destroy, .change_queue_depth = ufshcd_change_queue_depth, .eh_abort_handler = ufshcd_abort, .eh_device_reset_handler = ufshcd_eh_device_reset_handler, .eh_host_reset_handler = ufshcd_eh_host_reset_handler, .this_id = -1, .sg_tablesize = SG_ALL, .cmd_per_lun = UFSHCD_CMD_PER_LUN, .can_queue = UFSHCD_CAN_QUEUE, }; /** * ufshcd_suspend - suspend power management function * @hba: per adapter instance * @state: power state * * Returns -ENOSYS */ int ufshcd_suspend(struct ufs_hba *hba, pm_message_t state) { /* * TODO: * 1. Block SCSI requests from SCSI midlayer * 2. Change the internal driver state to non operational * 3. Set UTRLRSR and UTMRLRSR bits to zero * 4. Wait until outstanding commands are completed * 5. Set HCE to zero to send the UFS host controller to reset state */ return -ENOSYS; } EXPORT_SYMBOL_GPL(ufshcd_suspend); /** * ufshcd_resume - resume power management function * @hba: per adapter instance * * Returns -ENOSYS */ int ufshcd_resume(struct ufs_hba *hba) { /* * TODO: * 1. Set HCE to 1, to start the UFS host controller * initialization process * 2. Set UTRLRSR and UTMRLRSR bits to 1 * 3. Change the internal driver state to operational * 4. Unblock SCSI requests from SCSI midlayer */ return -ENOSYS; } EXPORT_SYMBOL_GPL(ufshcd_resume); int ufshcd_runtime_suspend(struct ufs_hba *hba) { if (!hba) return 0; /* * The device is idle with no requests in the queue, * allow background operations. */ return ufshcd_enable_auto_bkops(hba); } EXPORT_SYMBOL(ufshcd_runtime_suspend); int ufshcd_runtime_resume(struct ufs_hba *hba) { if (!hba) return 0; return ufshcd_disable_auto_bkops(hba); } EXPORT_SYMBOL(ufshcd_runtime_resume); int ufshcd_runtime_idle(struct ufs_hba *hba) { return 0; } EXPORT_SYMBOL(ufshcd_runtime_idle); /** * ufshcd_remove - de-allocate SCSI host and host memory space * data structure memory * @hba - per adapter instance */ void ufshcd_remove(struct ufs_hba *hba) { scsi_remove_host(hba->host); /* disable interrupts */ ufshcd_disable_intr(hba, hba->intr_mask); ufshcd_hba_stop(hba); scsi_host_put(hba->host); } EXPORT_SYMBOL_GPL(ufshcd_remove); /** * ufshcd_init - Driver initialization routine * @dev: pointer to device handle * @hba_handle: driver private handle * @mmio_base: base register address * @irq: Interrupt line of device * Returns 0 on success, non-zero value on failure */ int ufshcd_init(struct device *dev, struct ufs_hba **hba_handle, void __iomem *mmio_base, unsigned int irq) { struct Scsi_Host *host; struct ufs_hba *hba; int err; if (!dev) { dev_err(dev, "Invalid memory reference for dev is NULL\n"); err = -ENODEV; goto out_error; } if (!mmio_base) { dev_err(dev, "Invalid memory reference for mmio_base is NULL\n"); err = -ENODEV; goto out_error; } host = scsi_host_alloc(&ufshcd_driver_template, sizeof(struct ufs_hba)); if (!host) { dev_err(dev, "scsi_host_alloc failed\n"); err = -ENOMEM; goto out_error; } hba = shost_priv(host); hba->host = host; hba->dev = dev; hba->mmio_base = mmio_base; hba->irq = irq; /* Read capabilities registers */ ufshcd_hba_capabilities(hba); /* Get UFS version supported by the controller */ hba->ufs_version = ufshcd_get_ufs_version(hba); /* Get Interrupt bit mask per version */ hba->intr_mask = ufshcd_get_intr_mask(hba); /* Allocate memory for host memory space */ err = ufshcd_memory_alloc(hba); if (err) { dev_err(hba->dev, "Memory allocation failed\n"); goto out_disable; } /* Configure LRB */ ufshcd_host_memory_configure(hba); host->can_queue = hba->nutrs; host->cmd_per_lun = hba->nutrs; host->max_id = UFSHCD_MAX_ID; host->max_lun = UFSHCD_MAX_LUNS; host->max_channel = UFSHCD_MAX_CHANNEL; host->unique_id = host->host_no; host->max_cmd_len = MAX_CDB_SIZE; /* Initailize wait queue for task management */ init_waitqueue_head(&hba->tm_wq); init_waitqueue_head(&hba->tm_tag_wq); /* Initialize work queues */ INIT_WORK(&hba->eh_work, ufshcd_err_handler); INIT_WORK(&hba->eeh_work, ufshcd_exception_event_handler); /* Initialize UIC command mutex */ mutex_init(&hba->uic_cmd_mutex); /* Initialize mutex for device management commands */ mutex_init(&hba->dev_cmd.lock); /* Initialize device management tag acquire wait queue */ init_waitqueue_head(&hba->dev_cmd.tag_wq); /* IRQ registration */ err = devm_request_irq(dev, irq, ufshcd_intr, IRQF_SHARED, UFSHCD, hba); if (err) { dev_err(hba->dev, "request irq failed\n"); goto out_disable; } /* Enable SCSI tag mapping */ err = scsi_init_shared_tag_map(host, host->can_queue); if (err) { dev_err(hba->dev, "init shared queue failed\n"); goto out_disable; } err = scsi_add_host(host, hba->dev); if (err) { dev_err(hba->dev, "scsi_add_host failed\n"); goto out_disable; } /* Host controller enable */ err = ufshcd_hba_enable(hba); if (err) { dev_err(hba->dev, "Host controller enable failed\n"); goto out_remove_scsi_host; } *hba_handle = hba; /* Hold auto suspend until async scan completes */ pm_runtime_get_sync(dev); async_schedule(ufshcd_async_scan, hba); return 0; out_remove_scsi_host: scsi_remove_host(hba->host); out_disable: scsi_host_put(host); out_error: return err; } EXPORT_SYMBOL_GPL(ufshcd_init); MODULE_AUTHOR("Santosh Yaragnavi "); MODULE_AUTHOR("Vinayak Holikatti "); MODULE_DESCRIPTION("Generic UFS host controller driver Core"); MODULE_LICENSE("GPL"); MODULE_VERSION(UFSHCD_DRIVER_VERSION);