FUJITA Tomonori
2006-Jun-21 19:47 UTC
[Xen-devel] [PATCH] Virtual SCSI frontend and backend drivers
This patch contains virtual SCSI frontend and backend drivers. The frontend (initiator) and backend (target) drivers exchange SCSI RDMA Protocol (SRP) messages by using the ring queue like the blkfront and blkback drivers do. The backend driver exploits the SCSI target framework (tgt). tgt handles SCSI protocol (mostly in user-space). The tgt kernel component enables the backend driver to send the user-space daemon a vectors of commands through a netlink interface. The daemon executes the commands and send the results back to the backend driver. Currently, tgt enables you to emulate SCSI disk devices with regular and block device files. I can scan and detect disk with the drivers but they are far from completion, no error handling, no restore support, no clean shutdown. And I''m too lazy to implement SRP indirect data format so the drivers cannot handle large data transfer. The mm-kernels have the SRP helper library because there are other SRP drivers in mainline (IBM pServer initiator and target drivers and Infiniband initiator driver). It will simplify the Xen SCSI frontend and backend drivers greatly, but I''ve not integrated them. With the current tgt design, the daemon forces the backend driver to use page cache. So the backend driver does memory copies between domain0 and domainU. DIRECT_IO and small extension to tgt would avoid this problem. For further information about tgt: http://www.linuxsymposium.org/2006/view_abstract.php?content_key=19 Signed-off-by: FUJITA Tomonori <fujita.tomonori@lab.ntt.co.jp> --- diff -r 2a4377b4361d linux-2.6-xen-sparse/drivers/xen/Kconfig --- a/linux-2.6-xen-sparse/drivers/xen/Kconfig Sun Jun 18 19:26:22 2006 +0900 +++ b/linux-2.6-xen-sparse/drivers/xen/Kconfig Thu Jun 22 04:36:57 2006 +0900 @@ -81,6 +81,15 @@ config XEN_BLKDEV_BACKEND default y help The block-device backend driver allows the kernel to export its + block devices to other guests via a high-performance shared-memory + interface. + +config XEN_VSCSI_BACKEND + tristate "Virtual SCSI backend driver" + depends on XEN_BACKEND + default y + help + The virtual SCSI (SRP) backend driver allows the kernel to export its block devices to other guests via a high-performance shared-memory interface. @@ -153,6 +162,16 @@ config XEN_BLKDEV_FRONTEND dedicated device-driver domain, or your master control domain (domain 0), then you almost certainly want to say Y here. +config XEN_VSCSI_FRONTEND + tristate "Virtual SCSI frontend driver" + depends on XEN + default y + help + The virtual SCSI (SRP) frontend driver allows the kernel to access block + devices mounted within another guest OS. Unless you are building a + dedicated device-driver domain, or your master control domain + (domain 0), then you almost certainly want to say Y here. + config XEN_NETDEV_FRONTEND tristate "Network-device frontend driver" depends on XEN diff -r 2a4377b4361d linux-2.6-xen-sparse/drivers/xen/Makefile --- a/linux-2.6-xen-sparse/drivers/xen/Makefile Sun Jun 18 19:26:22 2006 +0900 +++ b/linux-2.6-xen-sparse/drivers/xen/Makefile Thu Jun 22 04:36:57 2006 +0900 @@ -12,9 +12,11 @@ obj-y += xenbus/ obj-$(CONFIG_XEN_BLKDEV_BACKEND) += blkback/ obj-$(CONFIG_XEN_NETDEV_BACKEND) += netback/ +obj-$(CONFIG_XEN_VSCSI_BACKEND) += vscsiback/ obj-$(CONFIG_XEN_TPMDEV_BACKEND) += tpmback/ obj-$(CONFIG_XEN_BLKDEV_FRONTEND) += blkfront/ obj-$(CONFIG_XEN_NETDEV_FRONTEND) += netfront/ +obj-$(CONFIG_XEN_VSCSI_FRONTEND) += vscsifront/ obj-$(CONFIG_XEN_BLKDEV_TAP) += blktap/ obj-$(CONFIG_XEN_TPMDEV_FRONTEND) += tpmfront/ obj-$(CONFIG_XEN_PCIDEV_BACKEND) += pciback/ diff -r 2a4377b4361d linux-2.6-xen-sparse/drivers/xen/vscsiback/Makefile --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/linux-2.6-xen-sparse/drivers/xen/vscsiback/Makefile Thu Jun 22 04:36:57 2006 +0900 @@ -0,0 +1,2 @@ +obj-$(CONFIG_XEN_VSCSI_BACKEND) += vscsibk.o +vscsibk-y += vscsiback.o interface.o libsrp.o diff -r 2a4377b4361d linux-2.6-xen-sparse/drivers/xen/vscsiback/interface.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/linux-2.6-xen-sparse/drivers/xen/vscsiback/interface.c Thu Jun 22 04:36:57 2006 +0900 @@ -0,0 +1,152 @@ +/****************************************************************************** + * arch/xen/drivers/blkif/backend/interface.c + * + * Block-device interface management. + * + * Copyright (c) 2004, Keir Fraser + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation; or, when distributed + * separately from the Linux kernel or incorporated into other + * software packages, subject to the following license: + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this source file (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, modify, + * merge, publish, distribute, sublicense, and/or sell copies of the Software, + * and to permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#include <linux/version.h> +#include <linux/sched.h> +#include <linux/interrupt.h> +#include <scsi/scsi.h> +#include <scsi/scsi_cmnd.h> +#include <scsi/scsi_host.h> +#include <scsi/scsi_device.h> +#include <scsi/srp.h> +#include <xen/driver_util.h> +#include <xen/evtchn.h> +#include <xen/xenbus.h> +#include <xen/interface/xen.h> +#include <xen/interface/io/vscsi.h> +#include <xen/interface/io/ring.h> +#include <xen/interface/grant_table.h> +#include <xen/gnttab.h> +#include <asm/hypervisor.h> +#include "vscsiback_priv.h" + +static int map_frontend_page(struct vscsiback_info *info, unsigned long shared_page) +{ + struct gnttab_map_grant_ref op; + int err; + + op.host_addr = (unsigned long)info->ring_area->addr; + op.flags = GNTMAP_host_map; + op.ref = shared_page; + op.dom = info->dev->otherend_id; + + lock_vm_area(info->ring_area); + err = HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref, &op, 1); + unlock_vm_area(info->ring_area); + BUG_ON(err); + + if (op.status) { + printk(" Grant table operation failure !\n"); + return op.status; + } + + info->shmem_ref = shared_page; + info->shmem_handle = op.handle; + +#ifdef CONFIG_XEN_IA64_DOM0_NON_VP + /* on some arch''s, map_grant_ref behaves like mmap, in that the + * passed address is a hint and a different address may be returned */ + info->ring_area->addr = gnttab_map_vaddr(op); +#endif + + return 0; +} + +static void unmap_frontend_page(struct vscsiback_info *info) +{ + struct gnttab_unmap_grant_ref op; + int err; + + op.host_addr = (unsigned long)info->ring_area->addr; + op.handle = info->shmem_handle; + op.dev_bus_addr = 0; + + lock_vm_area(info->ring_area); + err = HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref, &op, 1); + unlock_vm_area(info->ring_area); + BUG_ON(err); +} + +int vscsiback_init_sring(struct vscsiback_info *info, + unsigned long shared_page, unsigned int evtchn) +{ + struct vscsi_sring *sring; + int err; + evtchn_op_t op = { + .cmd = EVTCHNOP_bind_interdomain, + .u.bind_interdomain.remote_dom = info->dev->otherend_id, + .u.bind_interdomain.remote_port = evtchn }; + + if (info->irq) { + printk("Already connected through?\n"); + return 0; + } + + info->ring_area = alloc_vm_area(PAGE_SIZE); + if (!info) + return -ENOMEM; + + err = map_frontend_page(info, shared_page); + if (err) + goto free_vm; + + err = HYPERVISOR_event_channel_op(&op); + if (err) + goto unmap_page; + + info->evtchn = op.u.bind_interdomain.local_port; + + sring = (struct vscsi_sring *) info->ring_area->addr; + BACK_RING_INIT(&info->ring, sring, PAGE_SIZE); + + info->irq = bind_evtchn_to_irqhandler(info->evtchn, vscsiback_intr, + 0, "vscsi-backend", info); + return 0; + +unmap_page: + unmap_frontend_page(info); +free_vm: + free_vm_area(info->ring_area); + return err; +} + +void vscsiback_exit_sring(struct vscsiback_info *info) +{ + /* Already disconnected? */ + if (info->irq) + unbind_from_irqhandler(info->irq, info); + + if (info->ring.sring) { + unmap_frontend_page(info); + free_vm_area(info->ring_area); + } +} diff -r 2a4377b4361d linux-2.6-xen-sparse/drivers/xen/vscsiback/libsrp.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/linux-2.6-xen-sparse/drivers/xen/vscsiback/libsrp.c Thu Jun 22 04:36:57 2006 +0900 @@ -0,0 +1,259 @@ +/* + * SCSI RDAM Protocol lib functions + * + * Copyright (C) 2006 FUJITA Tomonori <tomof@acm.org> + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA + * 02110-1301 USA + */ +#include <linux/err.h> +#include <linux/kfifo.h> +#include <linux/scatterlist.h> +#include <linux/dma-mapping.h> +#include <scsi/scsi.h> +#include <scsi/scsi_cmnd.h> +#include <scsi/scsi_tcq.h> +#include <scsi/scsi_tgt.h> +#include <scsi/srp.h> +#include "libsrp.h" + +enum srp_task_attributes { + SRP_SIMPLE_TASK = 0, + SRP_HEAD_TASK = 1, + SRP_ORDERED_TASK = 2, + SRP_ACA_TASK = 4 +}; + +/* tmp - will replace with SCSI logging stuff */ +#define eprintk(fmt, args...) \ +do { \ + printk("%s(%d) " fmt, __FUNCTION__, __LINE__, ##args); \ +} while (0) +#define dprintk eprintk +/* #define dprintk(fmt, args...) */ + +static int direct_data(struct scsi_cmnd *sc, struct srp_direct_buf *md, + rdma_io_t rdma_io) +{ + struct scatterlist *sg = sc->request_buffer; + + dprintk("%u %u %d\n", sc->request_bufflen, md->len, sc->use_sg); + return rdma_io(sc, sg, sc->use_sg, md, 1, min(sc->request_bufflen, md->len)); +} + +#if 0 +static int indirect_data(struct scsi_cmnd *scmd, struct srp_cmd *cmd, + struct srp_indirect_buf *id, + enum dma_data_direction dir, rdma_io_t rdma_io) +{ + struct iu_entry *iue = (struct iu_entry *) scmd->SCp.ptr; + struct srp_target *target = iue->target; + struct srp_direct_buf *md; + struct scatterlist dummy, *sg = scmd->request_buffer; + dma_addr_t token = 0; + long err; + unsigned int done = 0; + int nmd, nsg; + + nmd = id->table_desc.len / sizeof(struct srp_direct_buf); + + dprintk("%p %u %u %u %u %d %d %d\n", + iue, scmd->request_bufflen, + id->len, scmd->offset, nmd, + cmd->data_in_desc_cnt, cmd->data_out_desc_cnt); + + if ((dir == DMA_FROM_DEVICE && nmd == cmd->data_in_desc_cnt) || + (dir == DMA_TO_DEVICE && nmd == cmd->data_out_desc_cnt)) { + md = &id->desc_list[0]; + goto rdma; + } + + md = dma_alloc_coherent(target->dev, id->table_desc.len, + &token, GFP_KERNEL); + if (!md) { + eprintk("Can''t get dma memory %u\n", id->table_desc.len); + return 0; + } + + sg_init_one(&dummy, md, id->table_desc.len); + sg_dma_address(&dummy) = token; + err = rdma_io(iue, &dummy, 1, &id->table_desc, 1, DMA_TO_DEVICE, + id->table_desc.len); + if (err < 0) { + eprintk("Error copying indirect table %ld\n", err); + goto free_mem; + } + +rdma: + nsg = dma_map_sg(target->dev, sg, scmd->use_sg, DMA_BIDIRECTIONAL); + if (!nsg) { + eprintk("fail to map %p %d\n", iue, scmd->use_sg); + goto free_mem; + } + + err = rdma_io(iue, sg, nsg, md, nmd, dir, + min(scmd->request_bufflen, id->len)); + dma_unmap_sg(target->dev, sg, nsg, DMA_BIDIRECTIONAL); + +free_mem: + if (token) + dma_free_coherent(target->dev, id->table_desc.len, md, token); + + return done; +} +#endif + +static int data_out_desc_size(struct srp_cmd *cmd) +{ + int size = 0; + u8 fmt = cmd->buf_fmt >> 4; + + switch (fmt) { + case SRP_NO_DATA_DESC: + break; + case SRP_DATA_DESC_DIRECT: + size = sizeof(struct srp_direct_buf); + break; + case SRP_DATA_DESC_INDIRECT: + size = sizeof(struct srp_indirect_buf) + + sizeof(struct srp_direct_buf) * cmd->data_out_desc_cnt; + break; + default: + eprintk("client error. Invalid data_out_format %x\n", fmt); + break; + } + return size; +} + +int srp_transfer_data(struct scsi_cmnd *sc, struct srp_cmd *cmd, + rdma_io_t rdma_io) +{ + struct srp_direct_buf *md; + struct srp_indirect_buf *id; + int offset, err = 0; + u8 format; + + offset = cmd->add_cdb_len * 4; + if (sc->sc_data_direction == DMA_FROM_DEVICE) + offset += data_out_desc_size(cmd); + + if (sc->sc_data_direction == DMA_TO_DEVICE) + format = cmd->buf_fmt >> 4; + else + format = cmd->buf_fmt & ((1U << 4) - 1); + + dprintk("%d %x %d\n", sc->sc_data_direction, format, offset); + + switch (format) { + case SRP_NO_DATA_DESC: + break; + case SRP_DATA_DESC_DIRECT: + md = (struct srp_direct_buf *) + (cmd->add_data + offset); + err = direct_data(sc, md, rdma_io); + break; + case SRP_DATA_DESC_INDIRECT: + id = (struct srp_indirect_buf *) + (cmd->add_data + offset); +/* err = indirect_data(sc, cmd, id, dir, rdma_io); */ + break; + default: + eprintk("Unknown format %d %x\n", sc->sc_data_direction, format); + break; + } + + return err; +} + +static int vscsis_data_length(struct srp_cmd *cmd, enum dma_data_direction dir) +{ + struct srp_direct_buf *md; + struct srp_indirect_buf *id; + int len = 0, offset = cmd->add_cdb_len * 4; + u8 fmt; + + if (dir == DMA_TO_DEVICE) + fmt = cmd->buf_fmt >> 4; + else { + fmt = cmd->buf_fmt & ((1U << 4) - 1); + offset += data_out_desc_size(cmd); + } + + switch (fmt) { + case SRP_NO_DATA_DESC: + break; + case SRP_DATA_DESC_DIRECT: + md = (struct srp_direct_buf *) (cmd->add_data + offset); + len = md->len; + break; + case SRP_DATA_DESC_INDIRECT: + id = (struct srp_indirect_buf *) (cmd->add_data + offset); + len = id->len; + break; + default: + eprintk("invalid data format %x\n", fmt); + break; + } + return len; +} + +int srp_cmd_perform(struct Scsi_Host *host, struct srp_cmd *cmd, void *data) +{ + enum dma_data_direction data_dir; + struct scsi_cmnd *sc; + int tag, len; + + tag = MSG_SIMPLE_TAG; + + switch (cmd->task_attr) { + case SRP_SIMPLE_TASK: + tag = MSG_SIMPLE_TAG; + break; + case SRP_ORDERED_TASK: + tag = MSG_ORDERED_TAG; + break; + case SRP_HEAD_TASK: + tag = MSG_HEAD_TAG; + break; + default: + eprintk("Task attribute %d not supported\n", cmd->task_attr); + tag = MSG_ORDERED_TAG; + } + + if (cmd->buf_fmt >> 4) + data_dir = DMA_TO_DEVICE; + else + data_dir = DMA_FROM_DEVICE; + len = vscsis_data_length(cmd, data_dir); + + dprintk("%x %llx %d %d %d %llx\n", cmd->cdb[0], + (unsigned long long) cmd->lun, data_dir, + len, tag, (unsigned long long) cmd->tag); + + sc = scsi_host_get_command(host, data_dir, GFP_KERNEL); + BUG_ON(!sc); + memcpy(sc->cmnd, cmd->cdb, MAX_COMMAND_SIZE); + sc->request_bufflen = len; + sc->tag = tag; + sc->SCp.ptr = data; + sc->host_scribble = (void *) host; + scsi_tgt_queue_command(sc, (struct scsi_lun *) &cmd->lun, cmd->tag); + + return 0; +} + +MODULE_DESCRIPTION("SCSI RDAM Protocol lib functions"); +MODULE_AUTHOR("FUJITA Tomonori"); +MODULE_LICENSE("GPL"); diff -r 2a4377b4361d linux-2.6-xen-sparse/drivers/xen/vscsiback/libsrp.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/linux-2.6-xen-sparse/drivers/xen/vscsiback/libsrp.h Thu Jun 22 04:36:57 2006 +0900 @@ -0,0 +1,15 @@ +#ifndef __LIBSRP_H__ +#define __LIBSRP_H__ + +#include <linux/list.h> +#include <scsi/scsi_cmnd.h> +#include <scsi/scsi_host.h> +#include <scsi/srp.h> + +typedef int (rdma_io_t) (struct scsi_cmnd *, struct scatterlist *, int, + struct srp_direct_buf *, int, unsigned int); + +extern int srp_cmd_perform(struct Scsi_Host *, struct srp_cmd *, void *); +extern int srp_transfer_data(struct scsi_cmnd *, struct srp_cmd *, rdma_io_t); + +#endif diff -r 2a4377b4361d linux-2.6-xen-sparse/drivers/xen/vscsiback/vscsiback.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/linux-2.6-xen-sparse/drivers/xen/vscsiback/vscsiback.c Thu Jun 22 04:36:57 2006 +0900 @@ -0,0 +1,384 @@ +/* + * Xen Virtual SCSI target driver + * + * Copyright (C) 2006 FUJITA Tomonori <tomof@acm.org> + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA + * 02110-1301 USA + */ + +#include <linux/version.h> +#include <linux/sched.h> +#include <linux/interrupt.h> +#include <scsi/scsi.h> +#include <scsi/scsi_cmnd.h> +#include <scsi/scsi_host.h> +#include <scsi/scsi_device.h> +#include <scsi/scsi_tgt.h> +#include <scsi/srp.h> +#include <xen/evtchn.h> +#include <xen/balloon.h> +#include <xen/xenbus.h> +#include <xen/interface/xen.h> +#include <xen/interface/io/vscsi.h> +#include <xen/interface/io/ring.h> +#include <xen/interface/grant_table.h> +#include <xen/gnttab.h> +#include <asm/hypervisor.h> +#include "vscsiback_priv.h" +#include "libsrp.h" + +#define eprintk(fmt, args...) \ +do { \ + printk("%s(%d) " fmt, __FUNCTION__, __LINE__, ##args); \ +} while (0) + +#define dprintk eprintk + +static unsigned int debug = 0; +module_param(debug, int, 0644); + +static struct workqueue_struct *vsbkd; + +static int req_index(struct vscsiback_info *info, struct vscsi_request *req) +{ + return (req - RING_GET_REQUEST(&info->ring, 0)); +} + +static int __idx(struct vscsiback_info *info, struct vscsi_request *req, + int idx) +{ + return req_index(info, req) * SRP_MAX_INDIRECT + idx; +} + +static unsigned long vaddr(struct vscsiback_info *info, + struct vscsi_request *req, int i) +{ + unsigned long vstart + (unsigned long) pfn_to_kaddr(page_to_pfn(info->page)); + + return vstart + (__idx(info, req, i) << PAGE_SHIFT); +} + +static int vscsiback_rdma(struct scsi_cmnd *sc, struct scatterlist *sg, int nsg, + struct srp_direct_buf *md, int nmd, + unsigned int rest) +{ + struct Scsi_Host *host = (struct Scsi_Host *) sc->host_scribble; + struct vscsiback_info *info = (struct vscsiback_info *) host->hostdata; + struct gnttab_map_grant_ref map[SRP_MAX_INDIRECT]; + struct gnttab_unmap_grant_ref unmap[SRP_MAX_INDIRECT]; + struct vscsi_request *req = (struct vscsi_request *) sc->SCp.ptr; + int i, err; + char *p, *q; + + for (i = 0; i < nmd; i++) { + map[i].host_addr = vaddr(info, req, i); + map[i].dom = info->dev->otherend_id; + map[i].ref = md[i].key; + map[i].flags = GNTMAP_host_map; + } + + err = HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref, map, nmd); + + for (i = 0; i < nmd; i++) + set_phys_to_machine(__pa(vaddr(info, req, i)) >> PAGE_SHIFT, + FOREIGN_FRAME(map[i].dev_bus_addr >> PAGE_SHIFT)); + + p = (char *) vaddr(info, req, 0) + (md->va & ~PAGE_MASK); + q = page_address(sg[0].page) + sg[0].offset; + if (sc->sc_data_direction == DMA_FROM_DEVICE) + memcpy(p, q, rest); + else + memcpy(q, p, rest); + + for (i = 0; i < nmd; i++) { + unmap[i].host_addr = vaddr(info, req, i); + unmap[i].dev_bus_addr = 0; + unmap[i].handle = map[i].handle; + } + + err = HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref, unmap, nmd); + return 0; +} + +static int vscsiback_transfer_data(struct scsi_cmnd *sc, + void (*done)(struct scsi_cmnd *)) +{ + struct vscsi_request *req = (struct vscsi_request *) sc->SCp.ptr; + int err; + + err = srp_transfer_data(sc, (struct srp_cmd *) req->buf, vscsiback_rdma); + done(sc); + return err; +} + +static int vscsiback_cmd_done(struct scsi_cmnd *sc, + void (*done)(struct scsi_cmnd *)) +{ + struct Scsi_Host *host = (struct Scsi_Host *) sc->host_scribble; + struct vscsiback_info *info = (struct vscsiback_info *) host->hostdata; + struct vscsi_back_ring *ring = &info->ring; + struct vscsi_response *rsp; + struct vscsi_request *req = (struct vscsi_request *) sc->SCp.ptr; + struct srp_cmd *cmd = (struct srp_cmd *) req->buf; + struct srp_rsp *srsp; + int notify; + + rsp = RING_GET_RESPONSE(ring, ring->rsp_prod_pvt); + srsp = (struct srp_rsp *) rsp->buf; + srsp->opcode = SRP_RSP; + srsp->tag = cmd->tag; + srsp->resp_data_len = 0; + srsp->status = NO_SENSE; + srsp->data_in_res_cnt = 0; + srsp->data_out_res_cnt = 0; + srsp->flags &= ~SRP_RSP_FLAG_RSPVALID; + + ring->rsp_prod_pvt++; + + RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(ring, notify); + + notify_remote_via_irq(info->irq); + + done(sc); + return 0; +} + +static int vscsiback_eh_abort_handler(struct scsi_cmnd *scmd) +{ + BUG_ON(1); + return 0; +} + +static struct scsi_host_template vscsiback_sht = { + .module = THIS_MODULE, + .name = "vscsiback", + .can_queue = 64, + .sg_tablesize = SG_ALL, + .use_clustering = DISABLE_CLUSTERING, + .transfer_response = vscsiback_cmd_done, + .transfer_data = vscsiback_transfer_data, + .eh_abort_handler = vscsiback_eh_abort_handler, +}; + +static void vscsiback_worker(void *data) +{ + struct vscsiback_info *info = data; + struct vscsi_back_ring *ring = &info->ring; + struct vscsi_request *req; + struct srp_cmd *cmd; + RING_IDX rc, rp; + + rc = ring->req_cons; + rp = ring->sring->req_prod; + rmb(); + + while ((rc != rp) && !RING_REQUEST_CONS_OVERFLOW(ring, rc)) { + eprintk("%u %u\n", rc, rp); + req = RING_GET_REQUEST(ring, rc); + ring->req_cons = ++rc; + cmd = (struct srp_cmd *) req->buf; + srp_cmd_perform(info->host, cmd, req); + } +} + +irqreturn_t vscsiback_intr(int irq, void *dev_id, struct pt_regs *regs) +{ + struct vscsiback_info *info = (struct vscsiback_info *) dev_id; + + queue_work(vsbkd, &info->vscsiback_work); + + return IRQ_HANDLED; +} + +static int vscsiback_connect(struct vscsiback_info *info) +{ + struct xenbus_device *dev = info->dev; + unsigned long ring_ref; + unsigned int evtchn; + int err; + + err = xenbus_gather(XBT_NULL, dev->otherend, "ring-ref", "%lu", + &ring_ref, "event-channel", "%u", &evtchn, NULL); + if (err) { + xenbus_dev_fatal(dev, err, "reading %s ring", dev->otherend); + return err; + } + + return vscsiback_init_sring(info, ring_ref, evtchn); +} + +/* static void vscsiback_frontend_changed(struct xenbus_device *dev, */ +/* enum xenbus_state frontend_state) */ +static void vscsiback_frontend_changed(struct xenbus_device *dev, + XenbusState frontend_state) +{ + struct vscsiback_info *info = dev->data; + int err; + + dprintk("%p %u %u\n", dev, dev->state, frontend_state); + switch (frontend_state) { + case XenbusStateInitialising: + break; + + case XenbusStateInitialised: + case XenbusStateConnected: + if (dev->state == XenbusStateConnected) + break; + + err = vscsiback_connect(info); + if (err) + break; + + err = xenbus_switch_state(dev, XenbusStateConnected); + if (err) + xenbus_dev_fatal(dev, err, "switching to Connected state", + dev->nodename); + break; + + case XenbusStateClosing: + xenbus_switch_state(dev, XenbusStateClosing); + break; + + case XenbusStateClosed: + device_unregister(&dev->dev); + break; + + case XenbusStateUnknown: + case XenbusStateInitWait: + default: + xenbus_dev_fatal(dev, -EINVAL, "saw state %d at frontend", + frontend_state); + break; + } +} + +static void vscsiback_backend_changed(struct xenbus_watch *watch, + const char **vec, unsigned int len) +{ + struct vscsiback_info *info + container_of(watch, struct vscsiback_info, backend_watch); + + dprintk("%p %u\n", info->dev, info->dev->state); + + /* TODO */ +} + +static int vscsiback_probe(struct xenbus_device *dev, + const struct xenbus_device_id *id) +{ + int err; + unsigned long nr = SRP_CAN_QUEUE * SRP_MAX_INDIRECT; + struct Scsi_Host *host; + struct vscsiback_info *info; + + dprintk("%p\n", dev); + + host = scsi_host_alloc(&vscsiback_sht, sizeof(struct vscsiback_info)); + if (!host) + return -ENOMEM; + err = scsi_tgt_alloc_queue(host); + if (err) + goto put_host; + + err = scsi_add_host(host, &dev->dev); + if (err) + goto put_host; + + info = (struct vscsiback_info *) host->hostdata; + dev->data = info; + info->dev = dev; + info->host = host; + + info->page = balloon_alloc_empty_page_range(nr); + if (!info->page) + goto put_host; + + INIT_WORK(&info->vscsiback_work, vscsiback_worker, info); + + err = xenbus_watch_path2(dev, dev->nodename, + "vscsi-host", + &info->backend_watch, + vscsiback_backend_changed); + if (err) + goto free_page; + + err = xenbus_switch_state(dev, XenbusStateInitWait); + if (err) + goto stop_watch; + + return 0; + +stop_watch: + /* free resource */ +free_page: + balloon_dealloc_empty_page_range(info->page, nr); +put_host: + scsi_host_put(host); + return err; +} + +static int vscsiback_remove(struct xenbus_device *dev) +{ + struct vscsiback_info *info = dev->data; + struct Scsi_Host *host = info->host; + + balloon_dealloc_empty_page_range(info->page, + SRP_CAN_QUEUE * SRP_MAX_INDIRECT); + scsi_remove_host(host); + scsi_host_put(host); + return 0; +} + +static struct xenbus_device_id vscsiback_ids[] = { + { "vscsi" }, + { "" } +}; + +static struct xenbus_driver vscsiback = { + .name = "vscsi", + .owner = THIS_MODULE, + .ids = vscsiback_ids, + .probe = vscsiback_probe, + .remove = vscsiback_remove, + .otherend_changed = vscsiback_frontend_changed +}; + +static int __init vscsiback_init(void) +{ + int err; + + if (xen_init() < 0) + return -ENODEV; + + vsbkd = create_singlethread_workqueue("vscsibkd"); + if (!vsbkd) + return -ENOMEM; + + err = xenbus_register_backend(&vscsiback); + if (err) + goto destroy_wq; + return 0; +destroy_wq: + destroy_workqueue(vsbkd); + return err; +} + +module_init(vscsiback_init); + +MODULE_AUTHOR("FUJITA Tomonori"); +MODULE_DESCRIPTION("Xen Virtual SCSI target driver"); +MODULE_LICENSE("GPL"); diff -r 2a4377b4361d linux-2.6-xen-sparse/drivers/xen/vscsiback/vscsiback_priv.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/linux-2.6-xen-sparse/drivers/xen/vscsiback/vscsiback_priv.h Thu Jun 22 04:36:57 2006 +0900 @@ -0,0 +1,23 @@ +struct vscsiback_info { + struct xenbus_device *dev; + struct Scsi_Host *host; + struct xenbus_watch backend_watch; + + unsigned int evtchn; + unsigned int irq; + + struct vscsi_back_ring ring; + struct vm_struct *ring_area; + + grant_handle_t shmem_handle; + grant_ref_t shmem_ref; + + struct work_struct vscsiback_work; + struct page *page; +}; + +extern irqreturn_t vscsiback_intr(int, void *, struct pt_regs *); + +extern int vscsiback_init_sring(struct vscsiback_info *, + unsigned long, unsigned int); +extern void vscsiback_exit_sring(struct vscsiback_info *); diff -r 2a4377b4361d linux-2.6-xen-sparse/drivers/xen/vscsifront/Makefile --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/linux-2.6-xen-sparse/drivers/xen/vscsifront/Makefile Thu Jun 22 04:36:57 2006 +0900 @@ -0,0 +1,1 @@ +obj-$(CONFIG_XEN_VSCSI_FRONTEND) += vscsifront.o diff -r 2a4377b4361d linux-2.6-xen-sparse/drivers/xen/vscsifront/vscsifront.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/linux-2.6-xen-sparse/drivers/xen/vscsifront/vscsifront.c Thu Jun 22 04:36:57 2006 +0900 @@ -0,0 +1,454 @@ +/* + * Xen Virtual SCSI initiator driver + * + * Copyright (C) 2006 FUJITA Tomonori <tomof@acm.org> + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA + * 02110-1301 USA + */ + +#include <linux/version.h> +#include <linux/sched.h> +#include <linux/interrupt.h> +#include <scsi/scsi.h> +#include <scsi/scsi_cmnd.h> +#include <scsi/scsi_host.h> +#include <scsi/scsi_device.h> +#include <scsi/srp.h> +#include <xen/evtchn.h> +#include <xen/xenbus.h> +#include <xen/interface/xen.h> +#include <xen/interface/io/vscsi.h> +#include <xen/interface/io/ring.h> +#include <xen/interface/grant_table.h> +#include <xen/gnttab.h> +#include <asm/hypervisor.h> + +#define eprintk(fmt, args...) \ +do { \ + printk("%s(%d) " fmt, __FUNCTION__, __LINE__, ##args); \ +} while (0) + +#define dprintk eprintk + +#define SRP_MAX_INDIRECT ((SRP_MAX_IU_LEN - \ + sizeof (struct srp_cmd) - \ + sizeof (struct srp_indirect_buf)) / 16) + +static unsigned int debug = 0; +module_param(debug, int, 0644); + +struct vscsifront_info { + struct xenbus_device *dev; + struct Scsi_Host *host; + unsigned int evtchn; + unsigned int irq; + unsigned long ring_ref; + struct vscsi_front_ring ring; +}; + +static int map_data_for_srp_cmd(struct vscsifront_info *info, + struct scsi_cmnd *sc, struct srp_cmd *cmd) +{ + struct scatterlist *sg = sc->request_buffer; + grant_ref_t gref_head; + int err, i, ref; + u8 fmt; + + if (!sg || sc->sc_data_direction == DMA_NONE) + return 0; + + err = gnttab_alloc_grant_references(SRP_MAX_INDIRECT, &gref_head); + if (err) + return -ENOMEM; + + if (sc->use_sg == 1) { + struct srp_direct_buf *buf = (void *) cmd->add_data; + fmt = SRP_DATA_DESC_DIRECT; + + ref = gnttab_claim_grant_reference(&gref_head); + gnttab_grant_foreign_access_ref(ref, info->dev->otherend_id, + page_to_phys(sg->page) >> PAGE_SHIFT, 0); + + buf->va = sg->offset; + buf->key = ref; + buf->len = sg->length; + + dprintk("%llx %u %u\n", (unsigned long long) buf->va, buf->key, + buf->len); + + } else { +/* struct srp_indirect_buf *buf = (void *) cmd->add_data; */ + fmt = SRP_DATA_DESC_INDIRECT; + + /* TODO */ + BUG(); + for (i = 0; i < sc->use_sg; i++) { + } + } + + if (sc->sc_data_direction == DMA_TO_DEVICE) + cmd->buf_fmt = fmt << 4; + else + cmd->buf_fmt = fmt; + + gnttab_free_grant_references(gref_head); + + return 0; +} + +static int vscsifront_queuecommand(struct scsi_cmnd *sc, + void (*done)(struct scsi_cmnd *)) +{ + struct Scsi_Host *host = sc->device->host; + struct vscsifront_info *info = (struct vscsifront_info *) host->hostdata; + struct vscsi_request *ring_req; + struct vscsi_front_ring *ring = &info->ring; + struct srp_cmd *cmd; + int err, notify; + + if (info->dev->state != XenbusStateConnected || RING_FULL(ring)) { + eprintk("busy %u!\n", info->dev->state); + return SCSI_MLQUEUE_HOST_BUSY; + } + sc->scsi_done = done; + sc->result = 0; + + ring_req = RING_GET_REQUEST(ring, ring->req_prod_pvt); + cmd = (struct srp_cmd *) ring_req->buf; + + memset(cmd, 0, SRP_MAX_IU_LEN); + cmd->opcode = SRP_CMD; + int_to_scsilun(sc->device->lun, (struct scsi_lun *) &cmd->lun); + cmd->tag = (long) sc; + memcpy(cmd->cdb, sc->cmnd, sc->cmd_len); + + err = map_data_for_srp_cmd(info, sc, cmd); + if (err) + return SCSI_MLQUEUE_HOST_BUSY; + + ring->req_prod_pvt++; + + RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(ring, notify); + notify_remote_via_irq(info->irq); + + return 0; +} + +static int vscsifront_eh_abort_handler(struct scsi_cmnd *sc) +{ + BUG(); + return 0; +} + +static void vscsifront_cmd_done(struct vscsifront_info *info, int idx) +{ + struct vscsi_front_ring *ring = &info->ring; + struct vscsi_request *ring_req; + struct srp_cmd *cmd; + struct scsi_cmnd *sc; + + ring_req = RING_GET_REQUEST(ring, idx); + cmd = (struct srp_cmd *) ring_req->buf; + sc = (struct scsi_cmnd *) (long) cmd->tag; + + if (!sc->request_buffer || (sc->sc_data_direction != DMA_TO_DEVICE && + sc->sc_data_direction != DMA_FROM_DEVICE)) + return; + + if (sc->use_sg == 1) { + struct srp_direct_buf *buf; + buf = (void *) cmd->add_data; + gnttab_end_foreign_access(buf->key, 0, 0UL); + } else { + /* TODO */ + } +} + +static irqreturn_t vscsifront_intr(int irq, void *dev_id, + struct pt_regs *ptregs) +{ + struct vscsifront_info *info = (struct vscsifront_info *) dev_id; + struct vscsi_front_ring *ring = &info->ring; + struct vscsi_response *ring_res; + struct scsi_cmnd *sc; + struct srp_rsp *rsp; + int i, rp; + + if (info->dev->state != XenbusStateConnected) + return IRQ_HANDLED; + +again: + rp = info->ring.sring->rsp_prod; + rmb(); + + for (i = info->ring.rsp_cons; i != rp; i++) { + ring_res = RING_GET_RESPONSE(ring, i); + + rsp = (struct srp_rsp *) ring_res->buf; + sc = ((void *) (unsigned long) rsp->tag); + sc->result = rsp->status; + + vscsifront_cmd_done(info, i); + + if (rsp->flags & SRP_RSP_FLAG_SNSVALID) { + memcpy(sc->sense_buffer, rsp->data + + be32_to_cpu(rsp->resp_data_len), + min_t(int, be32_to_cpu(rsp->sense_data_len), + SCSI_SENSE_BUFFERSIZE)); + } + + if (rsp->flags & (SRP_RSP_FLAG_DOOVER | SRP_RSP_FLAG_DOUNDER)) + sc->resid = be32_to_cpu(rsp->data_out_res_cnt); + else if (rsp->flags & (SRP_RSP_FLAG_DIOVER | SRP_RSP_FLAG_DIUNDER)) + sc->resid = be32_to_cpu(rsp->data_in_res_cnt); + + sc->scsi_done(sc); + } + + info->ring.rsp_cons = i; + if (i != info->ring.req_prod_pvt) { + int more_to_do; + RING_FINAL_CHECK_FOR_RESPONSES(ring, more_to_do); + if (more_to_do) + goto again; + } else + ring->sring->rsp_event = i + 1; + + return IRQ_HANDLED; +} + +static int vscsifront_alloc_ring(struct vscsifront_info *info) +{ + struct xenbus_device *dev = info->dev; + struct vscsi_sring *sring; + int err = -ENOMEM; + + sring = (struct vscsi_sring *) __get_free_page(GFP_KERNEL); + if (!sring) { + xenbus_dev_fatal(dev, err, "fail to allocate shared ring"); + return err; + } + + SHARED_RING_INIT(sring); + FRONT_RING_INIT(&info->ring, sring, PAGE_SIZE); + dprintk("%u\n", RING_SIZE(&info->ring)); + + err = xenbus_grant_ring(dev, virt_to_mfn(info->ring.sring)); + if (err < 0) { + xenbus_dev_fatal(dev, err, "fail to grant shared ring"); + goto free_sring; + } + info->ring_ref = err; + + err = xenbus_alloc_evtchn(dev, &info->evtchn); + if (err) { + xenbus_dev_fatal(dev, err, "fail to allocate evtchn"); + return err; + } + + err = bind_evtchn_to_irqhandler(info->evtchn, vscsifront_intr, + SA_SAMPLE_RANDOM, "vscsifront", info); + if (err <= 0) { + xenbus_dev_fatal(dev, err, "bind_evtchn_to_irqhandler failed"); + goto fail; + } + info->irq = err; + + return 0; +fail: + /* free resource */ +free_sring: + free_page((unsigned long) sring); + + return err; +} + +static int vscsifront_init_ring(struct vscsifront_info *info) +{ + struct xenbus_device *dev = info->dev; + xenbus_transaction_t xbt; + int err; + + dprintk(""); + + err = vscsifront_alloc_ring(info); + if (err) + return err; + dprintk("%lu %u\n", info->ring_ref, info->evtchn); + +again: + err = xenbus_transaction_start(&xbt); + if (err) { + xenbus_dev_fatal(dev, err, "starting transaction"); + } + + err = xenbus_printf(xbt, dev->nodename, "ring-ref", "%lu", + info->ring_ref); + if (err) { + xenbus_dev_fatal(dev, err, "%s", "writing ring-ref"); + goto fail; + } + + err = xenbus_printf(xbt, dev->nodename, "event-channel", "%u", + info->evtchn); + if (err) { + xenbus_dev_fatal(dev, err, "%s", "writing event-channel"); + goto fail; + } + + err = xenbus_transaction_end(xbt, 0); + if (err) { + if (err == -EAGAIN) + goto again; + xenbus_dev_fatal(dev, err, "completing transaction"); + } else + xenbus_switch_state(dev, XenbusStateInitialised); + + return 0; +fail: + xenbus_transaction_end(xbt, 1); + /* free resource */ + return err; +} + +static struct scsi_host_template vscsifront_sht = { + .module = THIS_MODULE, + .name = "vscsifront", + .queuecommand = vscsifront_queuecommand, + .eh_abort_handler = vscsifront_eh_abort_handler, + .cmd_per_lun = 16, + .can_queue = 8, + .this_id = -1, + .sg_tablesize = SRP_MAX_INDIRECT, + .use_clustering = ENABLE_CLUSTERING, +}; + +static int vscsifront_probe(struct xenbus_device *dev, + const struct xenbus_device_id *id) +{ + struct Scsi_Host *host; + struct vscsifront_info *info; + int err = -ENOMEM; + + host = scsi_host_alloc(&vscsifront_sht, sizeof(*info)); + if (!host) { + xenbus_dev_fatal(dev, err, "fail to allocate scsi host"); + return err; + } + info = (struct vscsifront_info *) host->hostdata; + dev->data = info; + info->dev = dev; + info->host = host; + + err = vscsifront_init_ring(info); + if (err) + scsi_host_put(host); + + return err; +} + +static int vscsifront_connect(struct vscsifront_info *info) +{ + struct xenbus_device *dev = info->dev; + struct Scsi_Host *host = info->host; + int err = -ENOMEM; + + dprintk("%u\n", dev->state); + if (dev->state == XenbusStateConnected) + return 0; + + xenbus_switch_state(dev, XenbusStateConnected); + + host->max_id = 1; + host->max_channel = 0; + + err = scsi_add_host(host, &dev->dev); + if (err) { + eprintk("fail to add scsi host %d\n", err); + return err; + } + scsi_scan_host(host); + + return 0; +} + +static int vscsifront_remove(struct xenbus_device *dev) +{ + struct vscsifront_info *info = dev->data; + + scsi_remove_host(info->host); + scsi_host_put(info->host); + + return 0; +} + +static void vscsifront_backend_changed(struct xenbus_device *dev, + XenbusState backend_state) +{ + struct vscsifront_info *info = dev->data; + + dprintk("%p %u %u\n", dev, dev->state, backend_state); + + switch (backend_state) { + case XenbusStateUnknown: + case XenbusStateInitialising: + case XenbusStateInitWait: + case XenbusStateInitialised: + case XenbusStateClosed: + break; + + case XenbusStateConnected: + vscsifront_connect(info); + break; + + case XenbusStateClosing: + break; + } +} + +static struct xenbus_device_id vscsifront_ids[] = { + { "vscsi" }, + { "" } +}; + +static struct xenbus_driver vscsifront = { + .name = "vscsi", + .owner = THIS_MODULE, + .ids = vscsifront_ids, + .probe = vscsifront_probe, + .remove = vscsifront_remove, +/* .resume = vscsifront_resume, */ + .otherend_changed = vscsifront_backend_changed, +}; + +static int __init vscsifront_init(void) +{ + if (xen_init() < 0) + return -ENODEV; + + return xenbus_register_frontend(&vscsifront); +} +static void vscsifront_exit(void) +{ + return xenbus_unregister_driver(&vscsifront); +} + +module_init(vscsifront_init); +module_exit(vscsifront_exit); + +MODULE_AUTHOR("FUJITA Tomonori"); +MODULE_DESCRIPTION("Xen Virtual SCSI initiator driver"); +MODULE_LICENSE("GPL"); diff -r 2a4377b4361d xen/include/public/io/vscsi.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/xen/include/public/io/vscsi.h Thu Jun 22 04:36:57 2006 +0900 @@ -0,0 +1,23 @@ +#ifndef __XEN__PUBLIC_IO_VSCSIIF_H__ +#define __XEN__PUBLIC_IO_VSCSIIF_H__ + +#include "ring.h" + +#define SRP_MAX_IU_LEN 256 +#define SRP_CAN_QUEUE 8 + +struct vscsi_request { + char buf[SRP_MAX_IU_LEN]; +}; + +struct vscsi_response { + char buf[sizeof(struct srp_rsp)]; +}; + +DEFINE_RING_TYPES(vscsi, struct vscsi_request, struct vscsi_response); + +#define SRP_MAX_INDIRECT ((SRP_MAX_IU_LEN - \ + sizeof (struct srp_cmd) - \ + sizeof (struct srp_indirect_buf)) / 16) + +#endif _______________________________________________ Xen-devel mailing list Xen-devel@lists.xensource.com http://lists.xensource.com/xen-devel