func: static void vhost_scsi_handle_kick(struct vhost_work *work) struct vhost_virtqueue * vq = container_of ( work , struct vhost_virtqueue poll . work struct vhost_scsi * vs = container_of ( vq -> dev , struct vhost_scsi , dev ) vhost_scsi_handle_vq ( vs , vq ); static vhost_scsi_handle_vq(struct vhost_scsi *vs, struct vhost_virtqueue *vq) struct tcm_vhost_tpg * * vs_tpg ; struct virtio_scsi_cmd_req v_req ; struct virtio_scsi_cmd_req_pi v_req_pi ; struct tcm_vhost_tpg * tpg ; struct tcm_vhost_cmd * cmd ; u64 tag ; u32 exp_data_len , data_first , data_num , data_direction , prot_first ; unsigned out , in , i ; int head , ret , data_niov , prot_niov , prot_bytes ; size_t req_size ; u16 lun ; u8 * target , * lunp , task_attr ; bool hdr_pi ; void * req , * cdb ; vs_tpg = vq -> private_data; if ( ! vs_tpg )  head = vhost_get_vq_desc ( vq , vq -> iov , ARRAY_SIZE ( vq -> iov ) , & out , & in , NULL , NULL ); if ( unlikely ( head < 0 ) )  if ( head == vq -> num )  if ( unlikely ( vhost_enable_notify ( & vs -> dev , vq ) ) )  if ( out == 1 && in == 1 )  data_direction = DMA_NONE; data_first = 0; data_num = 0; if ( out == 1 && in > 1 )  data_direction = DMA_FROM_DEVICE; data_first = out + 1; data_num = in - 1; if ( out > 1 && in == 1 )  data_direction = DMA_TO_DEVICE; data_first = 1; data_num = out - 1; if ( unlikely ( vq -> iov [ out ] . iov_len != sizeof ( struct virtio_scsi_cmd_resp ) ) )  if ( vhost_has_feature ( vq , VIRTIO_SCSI_F_T10_PI ) )  req = & v_req_pi; lunp = & v_req_pi . lun [ 0 ]; target = & v_req_pi . lun [ 1 ]; req_size = sizeof ( v_req_pi ); hdr_pi = true; req = & v_req; lunp = & v_req . lun [ 0 ]; target = & v_req . lun [ 1 ]; req_size = sizeof ( v_req ); hdr_pi = false; if ( unlikely ( vq -> iov [ 0 ] . iov_len < req_size ) )  ret = memcpy_fromiovecend ( req , & vq -> iov [ 0 ] , 0 , req_size ); if ( unlikely ( ret ) )  if ( unlikely ( * lunp != 1 ) )  tpg = ACCESS_ONCE ( vs_tpg [ * target ] ); if ( unlikely ( ! tpg ) )  data_niov = data_num; prot_niov = prot_first = prot_bytes = 0; if ( hdr_pi )  if ( v_req_pi . pi_bytesout )  if ( data_direction != DMA_TO_DEVICE )  prot_bytes = vhost32_to_cpu ( vq , v_req_pi . pi_bytesout ); if ( v_req_pi . pi_bytesin )  if ( data_direction != DMA_FROM_DEVICE )  prot_bytes = vhost32_to_cpu ( vq , v_req_pi . pi_bytesin ); if ( prot_bytes )  int tmp = 0 ; for (i = 0; i < data_num; i++) tmp += vq -> iov [ data_first + i ] . iov_len; prot_niov ++; if ( tmp >= prot_bytes )  prot_first = data_first; data_first += prot_niov; data_niov = data_num - prot_niov; tag = vhost64_to_cpu ( vq , v_req_pi . tag ); task_attr = v_req_pi . task_attr; cdb = & v_req_pi . cdb [ 0 ]; lun = ( ( v_req_pi . lun [ 2 ] << 8 ) | v_req_pi . lun [ 3 ] ) & 0x3FFF; tag = vhost64_to_cpu ( vq , v_req . tag ); task_attr = v_req . task_attr; cdb = & v_req . cdb [ 0 ]; lun = ( ( v_req . lun [ 2 ] << 8 ) | v_req . lun [ 3 ] ) & 0x3FFF; exp_data_len = 0; for (i = 0; i < data_niov; i++) exp_data_len += vq -> iov [ data_first + i ] . iov_len; if ( unlikely ( scsi_command_size ( cdb ) > TCM_VHOST_MAX_CDB_SIZE ) )  cmd = vhost_scsi_get_tag ( vq , tpg , cdb , tag , lun , task_attr , exp_data_len + prot_bytes , data_direction ); if ( IS_ERR ( cmd ) )  cmd -> tvc_vhost = vs; cmd -> tvc_vq = vq; cmd -> tvc_resp = vq -> iov [ out ] . iov_base; if ( prot_niov )  ret = vhost_scsi_map_iov_to_prot ( cmd , & vq -> iov [ prot_first ] , prot_niov , data_direction == DMA_FROM_DEVICE ); static vhost_scsi_map_iov_to_prot(struct tcm_vhost_cmd struct iovec int bool write) struct scatterlist * prot_sg = cmd -> tvc_prot_sgl ; unsigned int prot_sgl_count = 0 ; int ret , i ; for (i = 0; i < niov; i++) prot_sgl_count += iov_num_pages ( & iov [ i ] ); static int iov_num_pages(struct iovec *iov) return ( PAGE_ALIGN ( ( unsigned long ) iov -> iov_base + iov -> iov_len ) - ( ( unsigned long ) iov -> iov_base & PAGE_MASK ) ) >> PAGE_SHIFT ; if ( prot_sgl_count > TCM_VHOST_PREALLOC_PROT_SGLS )  return - ENOBUFS ; cmd -> tvc_prot_sgl_count = prot_sgl_count; for (i = 0; i < niov; i++) ret = vhost_scsi_map_to_sgl ( cmd , prot_sg , prot_sgl_count , & iov [ i ] , cmd -> tvc_upages , write ); static vhost_scsi_map_to_sgl(struct tcm_vhost_cmd struct scatterlist unsigned int struct iovec struct page bool write) void __user * ptr = iov -> iov_base ; int ret , i ; pages_nr = iov_num_pages ( iov ); static int iov_num_pages(struct iovec *iov) return ( PAGE_ALIGN ( ( unsigned long ) iov -> iov_base + iov -> iov_len ) - ( ( unsigned long ) iov -> iov_base & PAGE_MASK ) ) >> PAGE_SHIFT ; if ( pages_nr > sgl_count )  return - ENOBUFS ; if ( pages_nr > TCM_VHOST_PREALLOC_UPAGES )  return - ENOBUFS ; ret = get_user_pages_fast ( ( unsigned long ) ptr , pages_nr , write , pages ); if ( ret < 0 )  if ( ret != pages_nr )  ret = - EFAULT; return ret ; if ( ret < 0 )  return ret ; prot_sg += ret; prot_sgl_count -= ret; return 0 ; if ( unlikely ( ret ) )  if ( data_direction != DMA_NONE )  ret = vhost_scsi_map_iov_to_sgl ( cmd , & vq -> iov [ data_first ] , data_niov , data_direction == DMA_FROM_DEVICE ); static vhost_scsi_map_iov_to_sgl(struct tcm_vhost_cmd struct iovec int bool write) struct scatterlist * sg = cmd -> tvc_sgl ; unsigned int sgl_count = 0 ; int ret , i ; for (i = 0; i < niov; i++) sgl_count += iov_num_pages ( & iov [ i ] ); static int iov_num_pages(struct iovec *iov) return ( PAGE_ALIGN ( ( unsigned long ) iov -> iov_base + iov -> iov_len ) - ( ( unsigned long ) iov -> iov_base & PAGE_MASK ) ) >> PAGE_SHIFT ; if ( sgl_count > TCM_VHOST_PREALLOC_SGLS )  return - ENOBUFS ; cmd -> tvc_sgl_count = sgl_count; for (i = 0; i < niov; i++) ret = vhost_scsi_map_to_sgl ( cmd , sg , sgl_count , & iov [ i ] , cmd -> tvc_upages , write ); static vhost_scsi_map_to_sgl(struct tcm_vhost_cmd struct scatterlist unsigned int struct iovec struct page bool write) void __user * ptr = iov -> iov_base ; int ret , i ; pages_nr = iov_num_pages ( iov ); if ( pages_nr > sgl_count )  return - ENOBUFS ; if ( pages_nr > TCM_VHOST_PREALLOC_UPAGES )  return - ENOBUFS ; ret = get_user_pages_fast ( ( unsigned long ) ptr , pages_nr , write , pages ); if ( ret < 0 )  if ( ret != pages_nr )  ret = - EFAULT; return ret ; if ( ret < 0 )  return ret ; sg += ret; sgl_count -= ret; return 0 ; if ( unlikely ( ret ) )  static struct tcm_vhost_cmd vhost_scsi_get_tag(struct vhost_virtqueue *vq, struct tcm_vhost_tpg unsigned char *cdb, u64 scsi_tag, u16 lun, u8 u32 exp_data_len, int data_direction) struct tcm_vhost_cmd * cmd ; struct tcm_vhost_nexus * tv_nexus ; struct se_session * se_sess ; struct scatterlist * sg , * prot_sg ; struct page * * pages ; int tag ; tv_nexus = tpg -> tpg_nexus; if ( ! tv_nexus )  se_sess = tv_nexus -> tvn_se_sess; tag = percpu_ida_alloc ( & se_sess -> sess_tag_pool , TASK_RUNNING ); if ( tag < 0 )  cmd = & ( ( struct tcm_vhost_cmd * ) se_sess -> sess_cmd_map ) [ tag ]; sg = cmd -> tvc_sgl; prot_sg = cmd -> tvc_prot_sgl; pages = cmd -> tvc_upages; memset ( cmd , 0 , sizeof ( struct tcm_vhost_cmd ) ); cmd -> tvc_sgl = sg; cmd -> tvc_prot_sgl = prot_sg; cmd -> tvc_upages = pages; cmd -> tvc_se_cmd . map_tag = tag; cmd -> tvc_tag = scsi_tag; cmd -> tvc_lun = lun; cmd -> tvc_task_attr = task_attr; cmd -> tvc_exp_data_len = exp_data_len; cmd -> tvc_data_direction = data_direction; cmd -> tvc_nexus = tv_nexus; cmd -> inflight = tcm_vhost_get_inflight ( vq ); static struct vhost_scsi_inflight tcm_vhost_get_inflight(struct vhost_virtqueue *vq) struct vhost_scsi_inflight * inflight ; struct vhost_scsi_virtqueue * svq ; svq = container_of ( vq , struct vhost_scsi_virtqueue , vq ) inflight = & svq -> inflights [ svq -> inflight_idx ]; return inflight ; memcpy ( cmd -> tvc_cdb , cdb , TCM_VHOST_MAX_CDB_SIZE ); return cmd ; 