|
|
d89b3e |
From 1aefc4d0bab87ed9b1a531fad029cb0c24db220f Mon Sep 17 00:00:00 2001
|
|
|
d89b3e |
Message-Id: <1aefc4d0bab87ed9b1a531fad029cb0c24db220f@dist-git>
|
|
|
d89b3e |
From: Eric Blake <eblake@redhat.com>
|
|
|
d89b3e |
Date: Tue, 17 Mar 2015 15:12:49 -0600
|
|
|
d89b3e |
Subject: [PATCH] qemu: read backing chain names from qemu
|
|
|
d89b3e |
|
|
|
d89b3e |
7.1.z: https://bugzilla.redhat.com/show_bug.cgi?id=1203119
|
|
|
d89b3e |
7.2: https://bugzilla.redhat.com/show_bug.cgi?id=1199182
|
|
|
d89b3e |
|
|
|
d89b3e |
https://bugzilla.redhat.com/show_bug.cgi?id=1199182 documents that
|
|
|
d89b3e |
after a series of disk snapshots into existing destination images,
|
|
|
d89b3e |
followed by active commits of the top image, it is possible for
|
|
|
d89b3e |
qemu 2.2 and earlier to end up tracking a different name for the
|
|
|
d89b3e |
image than what it would have had when opening the chain afresh.
|
|
|
d89b3e |
That is, when starting with the chain 'a <- b <- c', the name
|
|
|
d89b3e |
associated with 'b' is how it was spelled in the metadata of 'c',
|
|
|
d89b3e |
but when starting with 'a', taking two snapshots into 'a <- b <- c',
|
|
|
d89b3e |
then committing 'c' back into 'b', the name associated with 'b' is
|
|
|
d89b3e |
now the name used when taking the first snapshot.
|
|
|
d89b3e |
|
|
|
d89b3e |
Sadly, older qemu doesn't know how to treat different spellings of
|
|
|
d89b3e |
the same filename as identical files (it uses strcmp() instead of
|
|
|
d89b3e |
checking for the same inode), which means libvirt's attempt to
|
|
|
d89b3e |
commit an image using solely the names learned from qcow2 metadata
|
|
|
d89b3e |
fails with a cryptic:
|
|
|
d89b3e |
|
|
|
d89b3e |
error: internal error: unable to execute QEMU command 'block-commit': Top image file /tmp/images/c/../b/b not found
|
|
|
d89b3e |
|
|
|
d89b3e |
even though the file exists. Trying to teach libvirt the rules on
|
|
|
d89b3e |
which name qemu will expect is not worth the effort (besides, we'd
|
|
|
d89b3e |
have to remember it across libvirtd restarts, and track whether a
|
|
|
d89b3e |
file was opened via metadata or via snapshot creation for a given
|
|
|
d89b3e |
qemu process); it is easier to just always directly ask qemu what
|
|
|
d89b3e |
string it expects to see in the first place.
|
|
|
d89b3e |
|
|
|
d89b3e |
As a safety valve, we validate that any name returned by qemu
|
|
|
d89b3e |
still maps to the same local file as we have tracked it, so that
|
|
|
d89b3e |
a compromised qemu cannot accidentally cause us to act on an
|
|
|
d89b3e |
incorrect file.
|
|
|
d89b3e |
|
|
|
d89b3e |
* src/qemu/qemu_monitor.h (qemuMonitorDiskNameLookup): New
|
|
|
d89b3e |
prototype.
|
|
|
d89b3e |
* src/qemu/qemu_monitor_json.h (qemuMonitorJSONDiskNameLookup):
|
|
|
d89b3e |
Likewise.
|
|
|
d89b3e |
* src/qemu/qemu_monitor.c (qemuMonitorDiskNameLookup): New function.
|
|
|
d89b3e |
* src/qemu/qemu_monitor_json.c (qemuMonitorJSONDiskNameLookup)
|
|
|
d89b3e |
(qemuMonitorJSONDiskNameLookupOne): Likewise.
|
|
|
d89b3e |
* src/qemu/qemu_driver.c (qemuDomainBlockCommit)
|
|
|
d89b3e |
(qemuDomainBlockJobImpl): Use it.
|
|
|
d89b3e |
|
|
|
d89b3e |
Signed-off-by: Eric Blake <eblake@redhat.com>
|
|
|
d89b3e |
(cherry picked from commit f9ea3d60119e82c02c00fbf3678c3ed20634dea1)
|
|
|
d89b3e |
|
|
|
d89b3e |
Conflicts:
|
|
|
d89b3e |
src/qemu/qemu_driver.c - context with older monitor wrap semantics
|
|
|
d89b3e |
Signed-off-by: Jiri Denemark <jdenemar@redhat.com>
|
|
|
d89b3e |
---
|
|
|
d89b3e |
src/qemu/qemu_driver.c | 28 ++++++------
|
|
|
d89b3e |
src/qemu/qemu_monitor.c | 18 ++++++++
|
|
|
d89b3e |
src/qemu/qemu_monitor.h | 8 +++-
|
|
|
d89b3e |
src/qemu/qemu_monitor_json.c | 102 ++++++++++++++++++++++++++++++++++++++++++-
|
|
|
d89b3e |
src/qemu/qemu_monitor_json.h | 9 +++-
|
|
|
d89b3e |
5 files changed, 148 insertions(+), 17 deletions(-)
|
|
|
d89b3e |
|
|
|
d89b3e |
diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c
|
|
|
d89b3e |
index 9dc243a..4293817 100644
|
|
|
d89b3e |
--- a/src/qemu/qemu_driver.c
|
|
|
d89b3e |
+++ b/src/qemu/qemu_driver.c
|
|
|
d89b3e |
@@ -15720,9 +15720,6 @@ qemuDomainBlockJobImpl(virDomainObjPtr vm,
|
|
|
d89b3e |
goto endjob;
|
|
|
d89b3e |
|
|
|
d89b3e |
if (baseSource) {
|
|
|
d89b3e |
- if (qemuGetDriveSourceString(baseSource, NULL, &basePath) < 0)
|
|
|
d89b3e |
- goto endjob;
|
|
|
d89b3e |
-
|
|
|
d89b3e |
if (flags & VIR_DOMAIN_BLOCK_REBASE_RELATIVE) {
|
|
|
d89b3e |
if (!virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_CHANGE_BACKING_FILE)) {
|
|
|
d89b3e |
virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
|
|
|
d89b3e |
@@ -15746,8 +15743,12 @@ qemuDomainBlockJobImpl(virDomainObjPtr vm,
|
|
|
d89b3e |
}
|
|
|
d89b3e |
|
|
|
d89b3e |
qemuDomainObjEnterMonitor(driver, vm);
|
|
|
d89b3e |
- ret = qemuMonitorBlockJob(priv->mon, device, basePath, backingPath,
|
|
|
d89b3e |
- bandwidth, info, mode, async);
|
|
|
d89b3e |
+ if (baseSource)
|
|
|
d89b3e |
+ basePath = qemuMonitorDiskNameLookup(priv->mon, device, disk->src,
|
|
|
d89b3e |
+ baseSource);
|
|
|
d89b3e |
+ if (!baseSource || basePath)
|
|
|
d89b3e |
+ ret = qemuMonitorBlockJob(priv->mon, device, basePath, backingPath,
|
|
|
d89b3e |
+ bandwidth, info, mode, async);
|
|
|
d89b3e |
qemuDomainObjExitMonitor(driver, vm);
|
|
|
d89b3e |
if (info && info->type == VIR_DOMAIN_BLOCK_JOB_TYPE_COMMIT &&
|
|
|
d89b3e |
disk->mirrorJob == VIR_DOMAIN_BLOCK_JOB_TYPE_ACTIVE_COMMIT)
|
|
|
d89b3e |
@@ -16313,12 +16314,6 @@ qemuDomainBlockCommit(virDomainPtr dom,
|
|
|
d89b3e |
VIR_DISK_CHAIN_READ_WRITE) < 0))
|
|
|
d89b3e |
goto endjob;
|
|
|
d89b3e |
|
|
|
d89b3e |
- if (qemuGetDriveSourceString(topSource, NULL, &topPath) < 0)
|
|
|
d89b3e |
- goto endjob;
|
|
|
d89b3e |
-
|
|
|
d89b3e |
- if (qemuGetDriveSourceString(baseSource, NULL, &basePath) < 0)
|
|
|
d89b3e |
- goto endjob;
|
|
|
d89b3e |
-
|
|
|
d89b3e |
if (flags & VIR_DOMAIN_BLOCK_COMMIT_RELATIVE &&
|
|
|
d89b3e |
topSource != disk->src) {
|
|
|
d89b3e |
if (!virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_CHANGE_BACKING_FILE)) {
|
|
|
d89b3e |
@@ -16349,9 +16344,14 @@ qemuDomainBlockCommit(virDomainPtr dom,
|
|
|
d89b3e |
disk->mirrorJob = VIR_DOMAIN_BLOCK_JOB_TYPE_ACTIVE_COMMIT;
|
|
|
d89b3e |
}
|
|
|
d89b3e |
qemuDomainObjEnterMonitor(driver, vm);
|
|
|
d89b3e |
- ret = qemuMonitorBlockCommit(priv->mon, device,
|
|
|
d89b3e |
- topPath, basePath, backingPath,
|
|
|
d89b3e |
- bandwidth);
|
|
|
d89b3e |
+ basePath = qemuMonitorDiskNameLookup(priv->mon, device, disk->src,
|
|
|
d89b3e |
+ baseSource);
|
|
|
d89b3e |
+ topPath = qemuMonitorDiskNameLookup(priv->mon, device, disk->src,
|
|
|
d89b3e |
+ topSource);
|
|
|
d89b3e |
+ if (basePath && topPath)
|
|
|
d89b3e |
+ ret = qemuMonitorBlockCommit(priv->mon, device,
|
|
|
d89b3e |
+ topPath, basePath, backingPath,
|
|
|
d89b3e |
+ bandwidth);
|
|
|
d89b3e |
qemuDomainObjExitMonitor(driver, vm);
|
|
|
d89b3e |
|
|
|
d89b3e |
if (ret == 0)
|
|
|
d89b3e |
diff --git a/src/qemu/qemu_monitor.c b/src/qemu/qemu_monitor.c
|
|
|
d89b3e |
index 0b1b80e..2bb6fdb 100644
|
|
|
d89b3e |
--- a/src/qemu/qemu_monitor.c
|
|
|
d89b3e |
+++ b/src/qemu/qemu_monitor.c
|
|
|
d89b3e |
@@ -3464,6 +3464,24 @@ qemuMonitorSupportsActiveCommit(qemuMonitorPtr mon)
|
|
|
d89b3e |
}
|
|
|
d89b3e |
|
|
|
d89b3e |
|
|
|
d89b3e |
+/* Determine the name that qemu is using for tracking the backing
|
|
|
d89b3e |
+ * element TARGET within the chain starting at TOP. */
|
|
|
d89b3e |
+char *
|
|
|
d89b3e |
+qemuMonitorDiskNameLookup(qemuMonitorPtr mon,
|
|
|
d89b3e |
+ const char *device,
|
|
|
d89b3e |
+ virStorageSourcePtr top,
|
|
|
d89b3e |
+ virStorageSourcePtr target)
|
|
|
d89b3e |
+{
|
|
|
d89b3e |
+ if (!mon->json) {
|
|
|
d89b3e |
+ virReportError(VIR_ERR_OPERATION_UNSUPPORTED, "%s",
|
|
|
d89b3e |
+ _("JSON monitor is required"));
|
|
|
d89b3e |
+ return NULL;
|
|
|
d89b3e |
+ }
|
|
|
d89b3e |
+
|
|
|
d89b3e |
+ return qemuMonitorJSONDiskNameLookup(mon, device, top, target);
|
|
|
d89b3e |
+}
|
|
|
d89b3e |
+
|
|
|
d89b3e |
+
|
|
|
d89b3e |
/* Use the block-job-complete monitor command to pivot a block copy
|
|
|
d89b3e |
* job. */
|
|
|
d89b3e |
int
|
|
|
d89b3e |
diff --git a/src/qemu/qemu_monitor.h b/src/qemu/qemu_monitor.h
|
|
|
d89b3e |
index 8930744..df6a8c0 100644
|
|
|
d89b3e |
--- a/src/qemu/qemu_monitor.h
|
|
|
d89b3e |
+++ b/src/qemu/qemu_monitor.h
|
|
|
d89b3e |
@@ -1,7 +1,7 @@
|
|
|
d89b3e |
/*
|
|
|
d89b3e |
* qemu_monitor.h: interaction with QEMU monitor console
|
|
|
d89b3e |
*
|
|
|
d89b3e |
- * Copyright (C) 2006-2014 Red Hat, Inc.
|
|
|
d89b3e |
+ * Copyright (C) 2006-2015 Red Hat, Inc.
|
|
|
d89b3e |
* Copyright (C) 2006 Daniel P. Berrange
|
|
|
d89b3e |
*
|
|
|
d89b3e |
* This library is free software; you can redistribute it and/or
|
|
|
d89b3e |
@@ -739,6 +739,12 @@ int qemuMonitorBlockCommit(qemuMonitorPtr mon,
|
|
|
d89b3e |
ATTRIBUTE_NONNULL(1) ATTRIBUTE_NONNULL(2) ATTRIBUTE_NONNULL(3)
|
|
|
d89b3e |
ATTRIBUTE_NONNULL(4);
|
|
|
d89b3e |
bool qemuMonitorSupportsActiveCommit(qemuMonitorPtr mon);
|
|
|
d89b3e |
+char *qemuMonitorDiskNameLookup(qemuMonitorPtr mon,
|
|
|
d89b3e |
+ const char *device,
|
|
|
d89b3e |
+ virStorageSourcePtr top,
|
|
|
d89b3e |
+ virStorageSourcePtr target)
|
|
|
d89b3e |
+ ATTRIBUTE_NONNULL(1) ATTRIBUTE_NONNULL(2) ATTRIBUTE_NONNULL(3)
|
|
|
d89b3e |
+ ATTRIBUTE_NONNULL(4);
|
|
|
d89b3e |
|
|
|
d89b3e |
int qemuMonitorArbitraryCommand(qemuMonitorPtr mon,
|
|
|
d89b3e |
const char *cmd,
|
|
|
d89b3e |
diff --git a/src/qemu/qemu_monitor_json.c b/src/qemu/qemu_monitor_json.c
|
|
|
d89b3e |
index 9b3d17a..e58c88b 100644
|
|
|
d89b3e |
--- a/src/qemu/qemu_monitor_json.c
|
|
|
d89b3e |
+++ b/src/qemu/qemu_monitor_json.c
|
|
|
d89b3e |
@@ -1,7 +1,7 @@
|
|
|
d89b3e |
/*
|
|
|
d89b3e |
* qemu_monitor_json.c: interaction with QEMU monitor console
|
|
|
d89b3e |
*
|
|
|
d89b3e |
- * Copyright (C) 2006-2014 Red Hat, Inc.
|
|
|
d89b3e |
+ * Copyright (C) 2006-2015 Red Hat, Inc.
|
|
|
d89b3e |
* Copyright (C) 2006 Daniel P. Berrange
|
|
|
d89b3e |
*
|
|
|
d89b3e |
* This library is free software; you can redistribute it and/or
|
|
|
d89b3e |
@@ -4221,6 +4221,106 @@ qemuMonitorJSONDrivePivot(qemuMonitorPtr mon, const char *device,
|
|
|
d89b3e |
}
|
|
|
d89b3e |
|
|
|
d89b3e |
|
|
|
d89b3e |
+static char *
|
|
|
d89b3e |
+qemuMonitorJSONDiskNameLookupOne(virJSONValuePtr image,
|
|
|
d89b3e |
+ virStorageSourcePtr top,
|
|
|
d89b3e |
+ virStorageSourcePtr target)
|
|
|
d89b3e |
+{
|
|
|
d89b3e |
+ virJSONValuePtr backing;
|
|
|
d89b3e |
+ char *ret;
|
|
|
d89b3e |
+
|
|
|
d89b3e |
+ /* The caller will report a generic message if we return NULL
|
|
|
d89b3e |
+ * without an error; but in some cases we can improve by reporting
|
|
|
d89b3e |
+ * a more specific message. */
|
|
|
d89b3e |
+ if (!top || !image)
|
|
|
d89b3e |
+ return NULL;
|
|
|
d89b3e |
+ if (top != target) {
|
|
|
d89b3e |
+ backing = virJSONValueObjectGet(image, "backing-image");
|
|
|
d89b3e |
+ return qemuMonitorJSONDiskNameLookupOne(backing, top->backingStore,
|
|
|
d89b3e |
+ target);
|
|
|
d89b3e |
+ }
|
|
|
d89b3e |
+ if (VIR_STRDUP(ret, virJSONValueObjectGetString(image, "filename")) < 0)
|
|
|
d89b3e |
+ return NULL;
|
|
|
d89b3e |
+ /* Sanity check - the name qemu gave us should resolve to the same
|
|
|
d89b3e |
+ file tracked by our target description. */
|
|
|
d89b3e |
+ if (virStorageSourceIsLocalStorage(target) &&
|
|
|
d89b3e |
+ STRNEQ(ret, target->path) &&
|
|
|
d89b3e |
+ !virFileLinkPointsTo(ret, target->path)) {
|
|
|
d89b3e |
+ virReportError(VIR_ERR_INTERNAL_ERROR,
|
|
|
d89b3e |
+ _("qemu block name '%s' doesn't match expected '%s'"),
|
|
|
d89b3e |
+ ret, target->path);
|
|
|
d89b3e |
+ VIR_FREE(ret);
|
|
|
d89b3e |
+ }
|
|
|
d89b3e |
+ return ret;
|
|
|
d89b3e |
+}
|
|
|
d89b3e |
+
|
|
|
d89b3e |
+
|
|
|
d89b3e |
+char *
|
|
|
d89b3e |
+qemuMonitorJSONDiskNameLookup(qemuMonitorPtr mon,
|
|
|
d89b3e |
+ const char *device,
|
|
|
d89b3e |
+ virStorageSourcePtr top,
|
|
|
d89b3e |
+ virStorageSourcePtr target)
|
|
|
d89b3e |
+{
|
|
|
d89b3e |
+ char *ret = NULL;
|
|
|
d89b3e |
+ virJSONValuePtr cmd = NULL;
|
|
|
d89b3e |
+ virJSONValuePtr reply = NULL;
|
|
|
d89b3e |
+ virJSONValuePtr devices;
|
|
|
d89b3e |
+ size_t i;
|
|
|
d89b3e |
+
|
|
|
d89b3e |
+ cmd = qemuMonitorJSONMakeCommand("query-block", NULL);
|
|
|
d89b3e |
+ if (!cmd)
|
|
|
d89b3e |
+ return NULL;
|
|
|
d89b3e |
+ if (qemuMonitorJSONCommand(mon, cmd, &reply) < 0)
|
|
|
d89b3e |
+ goto cleanup;
|
|
|
d89b3e |
+
|
|
|
d89b3e |
+ devices = virJSONValueObjectGet(reply, "return");
|
|
|
d89b3e |
+ if (!devices || devices->type != VIR_JSON_TYPE_ARRAY) {
|
|
|
d89b3e |
+ virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
|
|
|
d89b3e |
+ _("block info reply was missing device list"));
|
|
|
d89b3e |
+ goto cleanup;
|
|
|
d89b3e |
+ }
|
|
|
d89b3e |
+
|
|
|
d89b3e |
+ for (i = 0; i < virJSONValueArraySize(devices); i++) {
|
|
|
d89b3e |
+ virJSONValuePtr dev = virJSONValueArrayGet(devices, i);
|
|
|
d89b3e |
+ virJSONValuePtr inserted;
|
|
|
d89b3e |
+ virJSONValuePtr image;
|
|
|
d89b3e |
+ const char *thisdev;
|
|
|
d89b3e |
+
|
|
|
d89b3e |
+ if (!dev || dev->type != VIR_JSON_TYPE_OBJECT) {
|
|
|
d89b3e |
+ virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
|
|
|
d89b3e |
+ _("block info device entry was not in expected format"));
|
|
|
d89b3e |
+ goto cleanup;
|
|
|
d89b3e |
+ }
|
|
|
d89b3e |
+
|
|
|
d89b3e |
+ if (!(thisdev = virJSONValueObjectGetString(dev, "device"))) {
|
|
|
d89b3e |
+ virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
|
|
|
d89b3e |
+ _("block info device entry was not in expected format"));
|
|
|
d89b3e |
+ goto cleanup;
|
|
|
d89b3e |
+ }
|
|
|
d89b3e |
+
|
|
|
d89b3e |
+ if (STREQ(thisdev, device)) {
|
|
|
d89b3e |
+ if ((inserted = virJSONValueObjectGet(dev, "inserted")) &&
|
|
|
d89b3e |
+ (image = virJSONValueObjectGet(inserted, "image"))) {
|
|
|
d89b3e |
+ ret = qemuMonitorJSONDiskNameLookupOne(image, top, target);
|
|
|
d89b3e |
+ }
|
|
|
d89b3e |
+ break;
|
|
|
d89b3e |
+ }
|
|
|
d89b3e |
+ }
|
|
|
d89b3e |
+ /* Guarantee an error when returning NULL, but don't override a
|
|
|
d89b3e |
+ * more specific error if one was already generated. */
|
|
|
d89b3e |
+ if (!ret && !virGetLastError())
|
|
|
d89b3e |
+ virReportError(VIR_ERR_INTERNAL_ERROR,
|
|
|
d89b3e |
+ _("unable to find backing name for device %s"),
|
|
|
d89b3e |
+ device);
|
|
|
d89b3e |
+
|
|
|
d89b3e |
+ cleanup:
|
|
|
d89b3e |
+ virJSONValueFree(cmd);
|
|
|
d89b3e |
+ virJSONValueFree(reply);
|
|
|
d89b3e |
+
|
|
|
d89b3e |
+ return ret;
|
|
|
d89b3e |
+}
|
|
|
d89b3e |
+
|
|
|
d89b3e |
+
|
|
|
d89b3e |
int qemuMonitorJSONArbitraryCommand(qemuMonitorPtr mon,
|
|
|
d89b3e |
const char *cmd_str,
|
|
|
d89b3e |
char **reply_str,
|
|
|
d89b3e |
diff --git a/src/qemu/qemu_monitor_json.h b/src/qemu/qemu_monitor_json.h
|
|
|
d89b3e |
index ff20029..1028802 100644
|
|
|
d89b3e |
--- a/src/qemu/qemu_monitor_json.h
|
|
|
d89b3e |
+++ b/src/qemu/qemu_monitor_json.h
|
|
|
d89b3e |
@@ -1,7 +1,7 @@
|
|
|
d89b3e |
/*
|
|
|
d89b3e |
* qemu_monitor_json.h: interaction with QEMU monitor console
|
|
|
d89b3e |
*
|
|
|
d89b3e |
- * Copyright (C) 2006-2009, 2011-2014 Red Hat, Inc.
|
|
|
d89b3e |
+ * Copyright (C) 2006-2009, 2011-2015 Red Hat, Inc.
|
|
|
d89b3e |
* Copyright (C) 2006 Daniel P. Berrange
|
|
|
d89b3e |
*
|
|
|
d89b3e |
* This library is free software; you can redistribute it and/or
|
|
|
d89b3e |
@@ -280,6 +280,13 @@ int qemuMonitorJSONBlockCommit(qemuMonitorPtr mon,
|
|
|
d89b3e |
unsigned long long bandwidth)
|
|
|
d89b3e |
ATTRIBUTE_NONNULL(1) ATTRIBUTE_NONNULL(2);
|
|
|
d89b3e |
|
|
|
d89b3e |
+char *qemuMonitorJSONDiskNameLookup(qemuMonitorPtr mon,
|
|
|
d89b3e |
+ const char *device,
|
|
|
d89b3e |
+ virStorageSourcePtr top,
|
|
|
d89b3e |
+ virStorageSourcePtr target)
|
|
|
d89b3e |
+ ATTRIBUTE_NONNULL(1) ATTRIBUTE_NONNULL(2) ATTRIBUTE_NONNULL(3)
|
|
|
d89b3e |
+ ATTRIBUTE_NONNULL(4);
|
|
|
d89b3e |
+
|
|
|
d89b3e |
int qemuMonitorJSONArbitraryCommand(qemuMonitorPtr mon,
|
|
|
d89b3e |
const char *cmd_str,
|
|
|
d89b3e |
char **reply_str,
|
|
|
d89b3e |
--
|
|
|
d89b3e |
2.3.3
|
|
|
d89b3e |
|