Hi Maxime,
> -----Original Message-----
> From: Maxime Coquelin <maxime.coquelin@redhat.com>
> Sent: Monday, December 21, 2020 5:14 AM
> To: dev@dpdk.org; Xia, Chenbo <chenbo.xia@intel.com>; olivier.matz@6wind.com;
> amorenoz@redhat.com; david.marchand@redhat.com
> Cc: Maxime Coquelin <maxime.coquelin@redhat.com>
> Subject: [PATCH 28/40] net/virtio: add Virtio-user vring setting ops
>
> This patch introduces new callbacks for setting
> and getting vring state.
>
> Signed-off-by: Maxime Coquelin <maxime.coquelin@redhat.com>
<snip>
>
> @@ -327,6 +343,10 @@ struct virtio_user_backend_ops virtio_ops_vdpa = {
> .get_features = vhost_vdpa_get_features,
> .set_features = vhost_vdpa_set_features,
> .set_memory_table = vhost_vdpa_set_memory_table,
> + .set_vring_enable = vhost_vdpa_set_vring_enable,
Sorry, miss one comment in last email...
Do we still need to keep set_vring_enable in struct virtio_user_backend_ops? Because
as I notice, it's called only in another callback (enable_qp).
Thanks,
Chenbo
> + .set_vring_num = vhost_vdpa_set_vring_num,
> + .set_vring_base = vhost_vdpa_set_vring_base,
> + .get_vring_base = vhost_vdpa_get_vring_base,
> .send_request = vhost_vdpa_send_request,
> .enable_qp = vhost_vdpa_enable_queue_pair,
> .dma_map = vhost_vdpa_dma_map,
> diff --git a/drivers/net/virtio/virtio_user/virtio_user_dev.c
> b/drivers/net/virtio/virtio_user/virtio_user_dev.c
> index ae976be158..496a48ee51 100644
> --- a/drivers/net/virtio/virtio_user/virtio_user_dev.c
> +++ b/drivers/net/virtio/virtio_user/virtio_user_dev.c
> @@ -73,13 +73,13 @@ virtio_user_kick_queue(struct virtio_user_dev *dev,
> uint32_t queue_sel)
>
> state.index = queue_sel;
> state.num = vring->num;
> - dev->ops->send_request(dev, VHOST_USER_SET_VRING_NUM, &state);
> + dev->ops->set_vring_num(dev, &state);
>
> state.index = queue_sel;
> state.num = 0; /* no reservation */
> if (dev->features & (1ULL << VIRTIO_F_RING_PACKED))
> state.num |= (1 << 15);
> - dev->ops->send_request(dev, VHOST_USER_SET_VRING_BASE, &state);
> + dev->ops->set_vring_base(dev, &state);
>
> dev->ops->send_request(dev, VHOST_USER_SET_VRING_ADDR, &addr);
>
> @@ -218,9 +218,8 @@ int virtio_user_stop_device(struct virtio_user_dev *dev)
> /* Stop the backend. */
> for (i = 0; i < dev->max_queue_pairs * 2; ++i) {
> state.index = i;
> - if (dev->ops->send_request(dev, VHOST_USER_GET_VRING_BASE,
> - &state) < 0) {
> - PMD_DRV_LOG(ERR, "get_vring_base failed, index=%u\n",
> + if (dev->ops->get_vring_base(dev, &state) < 0) {
> + PMD_DRV_LOG(ERR, "get_vring_base failed, index=%u",
> i);
> error = -1;
> goto out;
> --
> 2.29.2
On 1/6/21 1:03 PM, Xia, Chenbo wrote:
> Hi Maxime,
>
>> -----Original Message-----
>> From: Maxime Coquelin <maxime.coquelin@redhat.com>
>> Sent: Monday, December 21, 2020 5:14 AM
>> To: dev@dpdk.org; Xia, Chenbo <chenbo.xia@intel.com>; olivier.matz@6wind.com;
>> amorenoz@redhat.com; david.marchand@redhat.com
>> Cc: Maxime Coquelin <maxime.coquelin@redhat.com>
>> Subject: [PATCH 28/40] net/virtio: add Virtio-user vring setting ops
>>
>> This patch introduces new callbacks for setting
>> and getting vring state.
>>
>> Signed-off-by: Maxime Coquelin <maxime.coquelin@redhat.com>
>
> <snip>
>
>>
>> @@ -327,6 +343,10 @@ struct virtio_user_backend_ops virtio_ops_vdpa = {
>> .get_features = vhost_vdpa_get_features,
>> .set_features = vhost_vdpa_set_features,
>> .set_memory_table = vhost_vdpa_set_memory_table,
>> + .set_vring_enable = vhost_vdpa_set_vring_enable,
>
> Sorry, miss one comment in last email...
>
> Do we still need to keep set_vring_enable in struct virtio_user_backend_ops? Because
> as I notice, it's called only in another callback (enable_qp).
Right, it can be dropped now. I'll remove it.
Thanks,
Maxime
> Thanks,
> Chenbo
>
>> + .set_vring_num = vhost_vdpa_set_vring_num,
>> + .set_vring_base = vhost_vdpa_set_vring_base,
>> + .get_vring_base = vhost_vdpa_get_vring_base,
>> .send_request = vhost_vdpa_send_request,
>> .enable_qp = vhost_vdpa_enable_queue_pair,
>> .dma_map = vhost_vdpa_dma_map,
>> diff --git a/drivers/net/virtio/virtio_user/virtio_user_dev.c
>> b/drivers/net/virtio/virtio_user/virtio_user_dev.c
>> index ae976be158..496a48ee51 100644
>> --- a/drivers/net/virtio/virtio_user/virtio_user_dev.c
>> +++ b/drivers/net/virtio/virtio_user/virtio_user_dev.c
>> @@ -73,13 +73,13 @@ virtio_user_kick_queue(struct virtio_user_dev *dev,
>> uint32_t queue_sel)
>>
>> state.index = queue_sel;
>> state.num = vring->num;
>> - dev->ops->send_request(dev, VHOST_USER_SET_VRING_NUM, &state);
>> + dev->ops->set_vring_num(dev, &state);
>>
>> state.index = queue_sel;
>> state.num = 0; /* no reservation */
>> if (dev->features & (1ULL << VIRTIO_F_RING_PACKED))
>> state.num |= (1 << 15);
>> - dev->ops->send_request(dev, VHOST_USER_SET_VRING_BASE, &state);
>> + dev->ops->set_vring_base(dev, &state);
>>
>> dev->ops->send_request(dev, VHOST_USER_SET_VRING_ADDR, &addr);
>>
>> @@ -218,9 +218,8 @@ int virtio_user_stop_device(struct virtio_user_dev *dev)
>> /* Stop the backend. */
>> for (i = 0; i < dev->max_queue_pairs * 2; ++i) {
>> state.index = i;
>> - if (dev->ops->send_request(dev, VHOST_USER_GET_VRING_BASE,
>> - &state) < 0) {
>> - PMD_DRV_LOG(ERR, "get_vring_base failed, index=%u\n",
>> + if (dev->ops->get_vring_base(dev, &state) < 0) {
>> + PMD_DRV_LOG(ERR, "get_vring_base failed, index=%u",
>> i);
>> error = -1;
>> goto out;
>> --
>> 2.29.2
>
@@ -107,6 +107,10 @@ struct virtio_user_backend_ops {
int (*get_protocol_features)(struct virtio_user_dev *dev, uint64_t *features);
int (*set_protocol_features)(struct virtio_user_dev *dev, uint64_t features);
int (*set_memory_table)(struct virtio_user_dev *dev);
+ int (*set_vring_enable)(struct virtio_user_dev *dev, struct vhost_vring_state *state);
+ int (*set_vring_num)(struct virtio_user_dev *dev, struct vhost_vring_state *state);
+ int (*set_vring_base)(struct virtio_user_dev *dev, struct vhost_vring_state *state);
+ int (*get_vring_base)(struct virtio_user_dev *dev, struct vhost_vring_state *state);
int (*send_request)(struct virtio_user_dev *dev,
enum vhost_user_request req,
void *arg);
@@ -219,12 +219,49 @@ vhost_kernel_set_memory_table(struct virtio_user_dev *dev)
return -1;
}
+static int
+vhost_kernel_set_vring(struct virtio_user_dev *dev, uint64_t req, struct vhost_vring_state *state)
+{
+ int ret, fd;
+ uint32_t index = state->index;
+
+ /* Convert from queue index to queue-pair & offset */
+ fd = dev->vhostfds[state->index / 2];
+ state->index %= 2;
+
+ ret = vhost_kernel_ioctl(fd, req, state);
+ if (ret < 0) {
+ PMD_DRV_LOG(ERR, "Failed to set vring (request %lu)", req);
+ return -1;
+ }
+
+ /* restore index back to queue index */
+ state->index = index;
+
+ return 0;
+}
+
+static int
+vhost_kernel_set_vring_num(struct virtio_user_dev *dev, struct vhost_vring_state *state)
+{
+ return vhost_kernel_set_vring(dev, VHOST_SET_VRING_NUM, state);
+}
+
+static int
+vhost_kernel_set_vring_base(struct virtio_user_dev *dev, struct vhost_vring_state *state)
+{
+ return vhost_kernel_set_vring(dev, VHOST_SET_VRING_BASE, state);
+}
+
+static int
+vhost_kernel_get_vring_base(struct virtio_user_dev *dev, struct vhost_vring_state *state)
+{
+ return vhost_kernel_set_vring(dev, VHOST_GET_VRING_BASE, state);
+}
+
static uint64_t vhost_req_user_to_kernel[] = {
[VHOST_USER_RESET_OWNER] = VHOST_RESET_OWNER,
[VHOST_USER_SET_VRING_CALL] = VHOST_SET_VRING_CALL,
- [VHOST_USER_SET_VRING_NUM] = VHOST_SET_VRING_NUM,
- [VHOST_USER_SET_VRING_BASE] = VHOST_SET_VRING_BASE,
- [VHOST_USER_GET_VRING_BASE] = VHOST_GET_VRING_BASE,
[VHOST_USER_SET_VRING_ADDR] = VHOST_SET_VRING_ADDR,
[VHOST_USER_SET_VRING_KICK] = VHOST_SET_VRING_KICK,
};
@@ -245,10 +282,7 @@ vhost_kernel_send_request(struct virtio_user_dev *dev,
req_kernel = vhost_req_user_to_kernel[req];
switch (req_kernel) {
- case VHOST_SET_VRING_NUM:
case VHOST_SET_VRING_ADDR:
- case VHOST_SET_VRING_BASE:
- case VHOST_GET_VRING_BASE:
case VHOST_SET_VRING_KICK:
case VHOST_SET_VRING_CALL:
queue_sel = *(unsigned int *)arg;
@@ -403,6 +437,9 @@ struct virtio_user_backend_ops virtio_ops_kernel = {
.get_features = vhost_kernel_get_features,
.set_features = vhost_kernel_set_features,
.set_memory_table = vhost_kernel_set_memory_table,
+ .set_vring_num = vhost_kernel_set_vring_num,
+ .set_vring_base = vhost_kernel_set_vring_base,
+ .get_vring_base = vhost_kernel_get_vring_base,
.send_request = vhost_kernel_send_request,
.enable_qp = vhost_kernel_enable_queue_pair
};
@@ -402,17 +402,94 @@ vhost_user_set_memory_table(struct virtio_user_dev *dev)
return -1;
}
+static int
+vhost_user_set_vring(struct virtio_user_dev *dev, enum vhost_user_request req,
+ struct vhost_vring_state *state)
+{
+ int ret;
+ struct vhost_user_msg msg = {
+ .request = req,
+ .flags = VHOST_USER_VERSION,
+ .size = sizeof(*state),
+ .payload.state = *state,
+ };
+
+ ret = vhost_user_write(dev->vhostfd, &msg, NULL, 0);
+ if (ret < 0) {
+ PMD_DRV_LOG(ERR, "Failed to set vring state (request %d)", req);
+ return -1;
+ }
+
+ return 0;
+}
+
+static int
+vhost_user_set_vring_enable(struct virtio_user_dev *dev, struct vhost_vring_state *state)
+{
+ return vhost_user_set_vring(dev, VHOST_USER_SET_VRING_ENABLE, state);
+}
+
+static int
+vhost_user_set_vring_num(struct virtio_user_dev *dev, struct vhost_vring_state *state)
+{
+ return vhost_user_set_vring(dev, VHOST_USER_SET_VRING_NUM, state);
+}
+
+static int
+vhost_user_set_vring_base(struct virtio_user_dev *dev, struct vhost_vring_state *state)
+{
+ return vhost_user_set_vring(dev, VHOST_USER_SET_VRING_BASE, state);
+}
+
+static int
+vhost_user_get_vring_base(struct virtio_user_dev *dev, struct vhost_vring_state *state)
+{
+ int ret;
+ struct vhost_user_msg msg;
+ unsigned int index = state->index;
+
+ ret = vhost_user_set_vring(dev, VHOST_USER_GET_VRING_BASE, state);
+ if (ret < 0) {
+ PMD_DRV_LOG(ERR, "Failed to send request");
+ goto err;
+ }
+
+ ret = vhost_user_read(dev->vhostfd, &msg);
+ if (ret < 0) {
+ PMD_DRV_LOG(ERR, "Failed to read reply");
+ goto err;
+ }
+
+ if (msg.request != VHOST_USER_GET_VRING_BASE) {
+ PMD_DRV_LOG(ERR, "Unexpected request type (%d)", msg.request);
+ goto err;
+ }
+
+ if (msg.size != sizeof(*state)) {
+ PMD_DRV_LOG(ERR, "Unexpected payload size (%u)", msg.size);
+ goto err;
+ }
+
+ if (msg.payload.state.index != index) {
+ PMD_DRV_LOG(ERR, "Unexpected ring index (%u)", state->index);
+ goto err;
+ }
+
+ *state = msg.payload.state;
+
+ return 0;
+err:
+ PMD_DRV_LOG(ERR, "Failed to get vring base");
+ return -1;
+}
+
static struct vhost_user_msg m;
const char * const vhost_msg_strings[] = {
[VHOST_USER_RESET_OWNER] = "VHOST_RESET_OWNER",
[VHOST_USER_SET_VRING_CALL] = "VHOST_SET_VRING_CALL",
- [VHOST_USER_SET_VRING_NUM] = "VHOST_SET_VRING_NUM",
- [VHOST_USER_SET_VRING_BASE] = "VHOST_SET_VRING_BASE",
- [VHOST_USER_GET_VRING_BASE] = "VHOST_GET_VRING_BASE",
[VHOST_USER_SET_VRING_ADDR] = "VHOST_SET_VRING_ADDR",
[VHOST_USER_SET_VRING_KICK] = "VHOST_SET_VRING_KICK",
- [VHOST_USER_SET_VRING_ENABLE] = "VHOST_SET_VRING_ENABLE",
[VHOST_USER_SET_STATUS] = "VHOST_SET_STATUS",
[VHOST_USER_GET_STATUS] = "VHOST_GET_STATUS",
};
@@ -474,19 +551,6 @@ vhost_user_sock(struct virtio_user_dev *dev,
fds[fd_num++] = *((int *)arg);
break;
- case VHOST_USER_SET_VRING_NUM:
- case VHOST_USER_SET_VRING_BASE:
- case VHOST_USER_SET_VRING_ENABLE:
- memcpy(&msg.payload.state, arg, sizeof(msg.payload.state));
- msg.size = sizeof(m.payload.state);
- break;
-
- case VHOST_USER_GET_VRING_BASE:
- memcpy(&msg.payload.state, arg, sizeof(msg.payload.state));
- msg.size = sizeof(m.payload.state);
- need_reply = 1;
- break;
-
case VHOST_USER_SET_VRING_ADDR:
memcpy(&msg.payload.addr, arg, sizeof(msg.payload.addr));
msg.size = sizeof(m.payload.addr);
@@ -535,14 +599,6 @@ vhost_user_sock(struct virtio_user_dev *dev,
}
*((__u64 *)arg) = msg.payload.u64;
break;
- case VHOST_USER_GET_VRING_BASE:
- if (msg.size != sizeof(m.payload.state)) {
- PMD_DRV_LOG(ERR, "Received bad msg size");
- return -1;
- }
- memcpy(arg, &msg.payload.state,
- sizeof(struct vhost_vring_state));
- break;
default:
/* Reply-ack handling */
if (msg.size != sizeof(m.payload.u64)) {
@@ -650,10 +706,10 @@ vhost_user_enable_queue_pair(struct virtio_user_dev *dev,
for (i = 0; i < 2; ++i) {
struct vhost_vring_state state = {
.index = pair_idx * 2 + i,
- .num = enable,
+ .num = enable,
};
- if (vhost_user_sock(dev, VHOST_USER_SET_VRING_ENABLE, &state))
+ if (vhost_user_set_vring_enable(dev, &state))
return -1;
}
@@ -669,6 +725,10 @@ struct virtio_user_backend_ops virtio_ops_user = {
.get_protocol_features = vhost_user_get_protocol_features,
.set_protocol_features = vhost_user_set_protocol_features,
.set_memory_table = vhost_user_set_memory_table,
+ .set_vring_enable = vhost_user_set_vring_enable,
+ .set_vring_num = vhost_user_set_vring_num,
+ .set_vring_base = vhost_user_set_vring_base,
+ .get_vring_base = vhost_user_get_vring_base,
.send_request = vhost_user_sock,
.enable_qp = vhost_user_enable_queue_pair
};
@@ -32,20 +32,15 @@
#define VHOST_VDPA_GET_DEVICE_ID _IOR(VHOST_VIRTIO, 0x70, __u32)
#define VHOST_VDPA_GET_STATUS _IOR(VHOST_VIRTIO, 0x71, __u8)
#define VHOST_VDPA_SET_STATUS _IOW(VHOST_VIRTIO, 0x72, __u8)
-#define VHOST_VDPA_SET_VRING_ENABLE _IOW(VHOST_VIRTIO, 0x75, \
- struct vhost_vring_state)
+#define VHOST_VDPA_SET_VRING_ENABLE _IOW(VHOST_VIRTIO, 0x75, struct vhost_vring_state)
static uint64_t vhost_req_user_to_vdpa[] = {
[VHOST_USER_RESET_OWNER] = VHOST_RESET_OWNER,
[VHOST_USER_SET_VRING_CALL] = VHOST_SET_VRING_CALL,
- [VHOST_USER_SET_VRING_NUM] = VHOST_SET_VRING_NUM,
- [VHOST_USER_SET_VRING_BASE] = VHOST_SET_VRING_BASE,
- [VHOST_USER_GET_VRING_BASE] = VHOST_GET_VRING_BASE,
[VHOST_USER_SET_VRING_ADDR] = VHOST_SET_VRING_ADDR,
[VHOST_USER_SET_VRING_KICK] = VHOST_SET_VRING_KICK,
[VHOST_USER_SET_STATUS] = VHOST_VDPA_SET_STATUS,
[VHOST_USER_GET_STATUS] = VHOST_VDPA_GET_STATUS,
- [VHOST_USER_SET_VRING_ENABLE] = VHOST_VDPA_SET_VRING_ENABLE,
};
/* no alignment requirement */
@@ -219,6 +214,30 @@ vhost_vdpa_set_memory_table(struct virtio_user_dev *dev)
return rte_memseg_walk_thread_unsafe(vhost_vdpa_map, dev);
}
+static int
+vhost_vdpa_set_vring_enable(struct virtio_user_dev *dev, struct vhost_vring_state *state)
+{
+ return vhost_vdpa_ioctl(dev->vhostfd, VHOST_VDPA_SET_VRING_ENABLE, state);
+}
+
+static int
+vhost_vdpa_set_vring_num(struct virtio_user_dev *dev, struct vhost_vring_state *state)
+{
+ return vhost_vdpa_ioctl(dev->vhostfd, VHOST_SET_VRING_NUM, state);
+}
+
+static int
+vhost_vdpa_set_vring_base(struct virtio_user_dev *dev, struct vhost_vring_state *state)
+{
+ return vhost_vdpa_ioctl(dev->vhostfd, VHOST_SET_VRING_BASE, state);
+}
+
+static int
+vhost_vdpa_get_vring_base(struct virtio_user_dev *dev, struct vhost_vring_state *state)
+{
+ return vhost_vdpa_ioctl(dev->vhostfd, VHOST_GET_VRING_BASE, state);
+}
+
/* with below features, vhost vdpa does not need to do the checksum and TSO,
* these info will be passed to virtio_user through virtio net header.
*/
@@ -247,10 +266,7 @@ vhost_vdpa_send_request(struct virtio_user_dev *dev,
req_vdpa = vhost_req_user_to_vdpa[req];
switch (req_vdpa) {
- case VHOST_SET_VRING_NUM:
case VHOST_SET_VRING_ADDR:
- case VHOST_SET_VRING_BASE:
- case VHOST_GET_VRING_BASE:
case VHOST_SET_VRING_KICK:
case VHOST_SET_VRING_CALL:
PMD_DRV_LOG(DEBUG, "vhostfd=%d, index=%u",
@@ -312,7 +328,7 @@ vhost_vdpa_enable_queue_pair(struct virtio_user_dev *dev,
.num = enable,
};
- if (vhost_vdpa_send_request(dev, VHOST_USER_SET_VRING_ENABLE, &state))
+ if (vhost_vdpa_set_vring_enable(dev, &state))
return -1;
}
@@ -327,6 +343,10 @@ struct virtio_user_backend_ops virtio_ops_vdpa = {
.get_features = vhost_vdpa_get_features,
.set_features = vhost_vdpa_set_features,
.set_memory_table = vhost_vdpa_set_memory_table,
+ .set_vring_enable = vhost_vdpa_set_vring_enable,
+ .set_vring_num = vhost_vdpa_set_vring_num,
+ .set_vring_base = vhost_vdpa_set_vring_base,
+ .get_vring_base = vhost_vdpa_get_vring_base,
.send_request = vhost_vdpa_send_request,
.enable_qp = vhost_vdpa_enable_queue_pair,
.dma_map = vhost_vdpa_dma_map,
@@ -73,13 +73,13 @@ virtio_user_kick_queue(struct virtio_user_dev *dev, uint32_t queue_sel)
state.index = queue_sel;
state.num = vring->num;
- dev->ops->send_request(dev, VHOST_USER_SET_VRING_NUM, &state);
+ dev->ops->set_vring_num(dev, &state);
state.index = queue_sel;
state.num = 0; /* no reservation */
if (dev->features & (1ULL << VIRTIO_F_RING_PACKED))
state.num |= (1 << 15);
- dev->ops->send_request(dev, VHOST_USER_SET_VRING_BASE, &state);
+ dev->ops->set_vring_base(dev, &state);
dev->ops->send_request(dev, VHOST_USER_SET_VRING_ADDR, &addr);
@@ -218,9 +218,8 @@ int virtio_user_stop_device(struct virtio_user_dev *dev)
/* Stop the backend. */
for (i = 0; i < dev->max_queue_pairs * 2; ++i) {
state.index = i;
- if (dev->ops->send_request(dev, VHOST_USER_GET_VRING_BASE,
- &state) < 0) {
- PMD_DRV_LOG(ERR, "get_vring_base failed, index=%u\n",
+ if (dev->ops->get_vring_base(dev, &state) < 0) {
+ PMD_DRV_LOG(ERR, "get_vring_base failed, index=%u",
i);
error = -1;
goto out;