[dpdk-dev] [PATCH v3] ethdev: extend flow metadata
Ori Kam
orika at mellanox.com
Sun Oct 27 17:56:18 CET 2019
Hi Slava,
Some small comments inline.
Please also add deprication for the META cap on TX.
> -----Original Message-----
> From: dev <dev-bounces at dpdk.org> On Behalf Of Viacheslav Ovsiienko
> Subject: [dpdk-dev] [PATCH v3] ethdev: extend flow metadata
>
> Currently, metadata can be set on egress path via mbuf tx_metadata field
> with PKT_TX_METADATA flag and RTE_FLOW_ITEM_TYPE_META matches
> metadata.
>
> This patch extends the metadata feature usability.
>
> 1) RTE_FLOW_ACTION_TYPE_SET_META
>
> When supporting multiple tables, Tx metadata can also be set by a rule and
> matched by another rule. This new action allows metadata to be set as a
> result of flow match.
>
> 2) Metadata on ingress
>
> There's also need to support metadata on ingress. Metadata can be set by
> SET_META action and matched by META item like Tx. The final value set by
> the action will be delivered to application via metadata dynamic field of
> mbuf which can be accessed by RTE_FLOW_DYNF_METADATA().
> PKT_RX_DYNF_METADATA flag will be set along with the data.
>
> The mbuf dynamic field must be registered by calling
> rte_flow_dynf_metadata_register() prior to use SET_META action.
>
> The availability of dynamic mbuf metadata field can be checked
> with rte_flow_dynf_metadata_avail() routine.
>
> For loopback/hairpin packet, metadata set on Rx/Tx may or may not be
> propagated to the other path depending on hardware capability.
>
> Signed-off-by: Yongseok Koh <yskoh at mellanox.com>
> Signed-off-by: Viacheslav Ovsiienko <viacheslavo at mellanox.com>
> ---
> v3: - removed MBUF_DYNF_METADATA_xxx definitions, only
> MBUF_DYNF_METADATA_NAME remains in rte_mbuf_dyn.h's
> centralizing point
> - added rte_flow_dynf_metadata_set/get helpers (Olivier)
> - updated rte_ethdev_version.map
> - name follows dynamic field name conventions
> - rebased
>
> v2: -
> https://eur03.safelinks.protection.outlook.com/?url=http%3A%2F%2Fpatches.dp
> dk.org%2Fpatch%2F60908%2F&data=02%7C01%7Corika%40mellanox.com
> %7C71a0c68aa47f482cf44c08d75883425e%7Ca652971c7d2e4d9ba6a4d149256
> f461b%7C0%7C0%7C637075193088507126&sdata=V4aOinownen1dLOddq
> pAzTp4GuTFL02BtMv08JCvZYY%3D&reserved=0
> - rebased
> - relies on dynamic mbuf field feature
>
> v1:
> https://eur03.safelinks.protection.outlook.com/?url=http%3A%2F%2Fpatches.dp
> dk.org%2Fpatch%2F56103%2F&data=02%7C01%7Corika%40mellanox.com
> %7C71a0c68aa47f482cf44c08d75883425e%7Ca652971c7d2e4d9ba6a4d149256
> f461b%7C0%7C0%7C637075193088507126&sdata=tznGLMEFGEqKsSnbjUd
> aQSHc%2BEh3%2FxTJwWt5fnZ7lK4%3D&reserved=0
>
> rfc:
> https://eur03.safelinks.protection.outlook.com/?url=http%3A%2F%2Fpatches.dp
> dk.org%2Fpatch%2F54270%2F&data=02%7C01%7Corika%40mellanox.com
> %7C71a0c68aa47f482cf44c08d75883425e%7Ca652971c7d2e4d9ba6a4d149256
> f461b%7C0%7C0%7C637075193088507126&sdata=rRu5Rt1AV14qXhMlPi5
> H%2FVSOM%2F3JGGNhVZw7P3SfuzY%3D&reserved=0
>
> app/test-pmd/cmdline_flow.c | 57 +++++++++++++++++-
> app/test-pmd/util.c | 5 ++
> doc/guides/prog_guide/rte_flow.rst | 72 ++++++++++++++++++-----
> doc/guides/rel_notes/release_19_11.rst | 15 +++++
> lib/librte_ethdev/rte_ethdev.h | 1 -
> lib/librte_ethdev/rte_ethdev_version.map | 3 +
> lib/librte_ethdev/rte_flow.c | 41 +++++++++++++
> lib/librte_ethdev/rte_flow.h | 99
> +++++++++++++++++++++++++++++++-
> lib/librte_mbuf/rte_mbuf_dyn.h | 8 ++-
> 9 files changed, 279 insertions(+), 22 deletions(-)
>
> diff --git a/app/test-pmd/cmdline_flow.c b/app/test-pmd/cmdline_flow.c
> index f48f4eb..bc89bf9 100644
> --- a/app/test-pmd/cmdline_flow.c
> +++ b/app/test-pmd/cmdline_flow.c
> @@ -308,6 +308,9 @@ enum index {
> ACTION_DEC_TCP_ACK_VALUE,
> ACTION_RAW_ENCAP,
> ACTION_RAW_DECAP,
> + ACTION_SET_META,
> + ACTION_SET_META_DATA,
> + ACTION_SET_META_MASK,
> };
>
> /** Maximum size for pattern in struct rte_flow_item_raw. */
> @@ -1005,6 +1008,7 @@ struct parse_action_priv {
> ACTION_DEC_TCP_ACK,
> ACTION_RAW_ENCAP,
> ACTION_RAW_DECAP,
> + ACTION_SET_META,
> ZERO,
> };
>
> @@ -1191,6 +1195,13 @@ struct parse_action_priv {
> ZERO,
> };
>
> +static const enum index action_set_meta[] = {
> + ACTION_SET_META_DATA,
> + ACTION_SET_META_MASK,
> + ACTION_NEXT,
> + ZERO,
> +};
> +
> static int parse_set_raw_encap_decap(struct context *, const struct token *,
> const char *, unsigned int,
> void *, unsigned int);
> @@ -1249,6 +1260,10 @@ static int parse_vc_action_raw_encap(struct
> context *,
> static int parse_vc_action_raw_decap(struct context *,
> const struct token *, const char *,
> unsigned int, void *, unsigned int);
> +static int parse_vc_action_set_meta(struct context *ctx,
> + const struct token *token, const char *str,
> + unsigned int len, void *buf,
> + unsigned int size);
> static int parse_destroy(struct context *, const struct token *,
> const char *, unsigned int,
> void *, unsigned int);
> @@ -3255,7 +3270,31 @@ static int comp_vc_action_rss_queue(struct context
> *, const struct token *,
> .help = "set raw decap data",
> .next = NEXT(next_item),
> .call = parse_set_raw_encap_decap,
> - }
> + },
> + [ACTION_SET_META] = {
> + .name = "set_meta",
> + .help = "set metadata",
> + .priv = PRIV_ACTION(SET_META,
> + sizeof(struct rte_flow_action_set_meta)),
> + .next = NEXT(action_set_meta),
> + .call = parse_vc_action_set_meta,
> + },
> + [ACTION_SET_META_DATA] = {
> + .name = "data",
> + .help = "metadata value",
> + .next = NEXT(action_set_meta, NEXT_ENTRY(UNSIGNED)),
> + .args = ARGS(ARGS_ENTRY_HTON
> + (struct rte_flow_action_set_meta, data)),
> + .call = parse_vc_conf,
> + },
> + [ACTION_SET_META_MASK] = {
> + .name = "mask",
> + .help = "mask for metadata value",
> + .next = NEXT(action_set_meta, NEXT_ENTRY(UNSIGNED)),
> + .args = ARGS(ARGS_ENTRY_HTON
> + (struct rte_flow_action_set_meta, mask)),
> + .call = parse_vc_conf,
> + },
> };
>
> /** Remove and return last entry from argument stack. */
> @@ -4625,6 +4664,22 @@ static int comp_vc_action_rss_queue(struct context
> *, const struct token *,
> return ret;
> }
>
> +static int
> +parse_vc_action_set_meta(struct context *ctx, const struct token *token,
> + const char *str, unsigned int len, void *buf,
> + unsigned int size)
> +{
> + int ret;
> +
> + ret = parse_vc(ctx, token, str, len, buf, size);
> + if (ret < 0)
> + return ret;
> + ret = rte_flow_dynf_metadata_register();
> + if (ret < 0)
> + return -1;
> + return len;
> +}
> +
> /** Parse tokens for destroy command. */
> static int
> parse_destroy(struct context *ctx, const struct token *token,
> diff --git a/app/test-pmd/util.c b/app/test-pmd/util.c
> index 1570270..39ff07b 100644
> --- a/app/test-pmd/util.c
> +++ b/app/test-pmd/util.c
> @@ -81,6 +81,11 @@
> mb->vlan_tci, mb->vlan_tci_outer);
> else if (ol_flags & PKT_RX_VLAN)
> printf(" - VLAN tci=0x%x", mb->vlan_tci);
> + if (ol_flags & PKT_TX_METADATA)
> + printf(" - Tx metadata: 0x%x", mb->tx_metadata);
> + if (ol_flags & PKT_RX_DYNF_METADATA)
> + printf(" - Rx metadata: 0x%x",
> + *RTE_FLOW_DYNF_METADATA(mb));
> if (mb->packet_type) {
> rte_get_ptype_name(mb->packet_type, buf,
> sizeof(buf));
> printf(" - hw ptype: %s", buf);
> diff --git a/doc/guides/prog_guide/rte_flow.rst
> b/doc/guides/prog_guide/rte_flow.rst
> index 6e6d44d..2b49baa 100644
> --- a/doc/guides/prog_guide/rte_flow.rst
> +++ b/doc/guides/prog_guide/rte_flow.rst
> @@ -658,6 +658,32 @@ the physical device, with virtual groups in the PMD or
> not at all.
> | ``mask`` | ``id`` | zeroed to match any value |
> +----------+----------+---------------------------+
>
> +Item: ``META``
> +^^^^^^^^^^^^^^^^^
> +
> +Matches 32 bit metadata item set.
> +
> +On egress, metadata can be set either by mbuf metadata field with
> +PKT_TX_METADATA flag or ``SET_META`` action. On ingress, ``SET_META``
> +action sets metadata for a packet and the metadata will be reported via
> +``metadata`` dynamic field of ``rte_mbuf`` with PKT_RX_DYNF_METADATA
> flag.
> +
> +- Default ``mask`` matches the specified Rx metadata value.
> +
> +.. _table_rte_flow_item_meta:
> +
> +.. table:: META
> +
> + +----------+----------+---------------------------------------+
> + | Field | Subfield | Value |
> +
> +==========+==========+=======================================+
> + | ``spec`` | ``data`` | 32 bit metadata value |
> + +----------+----------+---------------------------------------+
> + | ``last`` | ``data`` | upper range value |
I don't think this field should be used.
> + +----------+----------+---------------------------------------+
> + | ``mask`` | ``data`` | bit-mask applies to "spec" and "last" |
> + +----------+----------+---------------------------------------+
> +
> Data matching item types
> ~~~~~~~~~~~~~~~~~~~~~~~~
>
> @@ -1232,21 +1258,6 @@ Matches a PPPoE session protocol identifier.
> - ``proto_id``: PPP protocol identifier.
> - Default ``mask`` matches proto_id only.
>
Why this lines are changed?
> -
> -.. _table_rte_flow_item_meta:
> -
> -.. table:: META
> -
> - +----------+----------+---------------------------------------+
> - | Field | Subfield | Value |
> -
> +==========+==========+=======================================+
> - | ``spec`` | ``data`` | 32 bit metadata value |
> - +----------+--------------------------------------------------+
> - | ``last`` | ``data`` | upper range value |
> - +----------+----------+---------------------------------------+
> - | ``mask`` | ``data`` | bit-mask applies to "spec" and "last" |
> - +----------+----------+---------------------------------------+
> -
> Item: ``NSH``
> ^^^^^^^^^^^^^^^^^^^
>
> @@ -2466,6 +2477,37 @@ Value to decrease TCP acknowledgment number by
> is a big-endian 32 bit integer.
>
> Using this action on non-matching traffic will result in undefined behavior.
>
> +Action: ``SET_META``
> +^^^^^^^^^^^^^^^^^^^^^^^
> +
> +Set metadata. Item ``META`` matches metadata.
> +
> +Metadata set by mbuf metadata field with PKT_TX_METADATA flag on egress
> will be
> +overridden by this action. On ingress, the metadata will be carried by
> +``metadata`` dynamic field of ``rte_mbuf`` which can be accessed by
> +``RTE_FLOW_DYNF_METADATA()``. PKT_RX_DYNF_METADATA flag will be set
> along
> +with the data.
> +
> +The mbuf dynamic field must be registered by calling
> +``rte_flow_dynf_metadata_register()`` prior to use ``SET_META`` action.
> +
> +Altering partial bits is supported with ``mask``. For bits which have never been
> +set, unpredictable value will be seen depending on driver implementation. For
> +loopback/hairpin packet, metadata set on Rx/Tx may or may not be
> propagated to
> +the other path depending on HW capability.
> +
> +.. _table_rte_flow_action_set_meta:
> +
> +.. table:: SET_META
> +
> + +----------+----------------------------+
> + | Field | Value |
> + +==========+============================+
> + | ``data`` | 32 bit metadata value |
> + +----------+----------------------------+
> + | ``mask`` | bit-mask applies to "data" |
> + +----------+----------------------------+
> +
> Negative types
> ~~~~~~~~~~~~~~
>
> diff --git a/doc/guides/rel_notes/release_19_11.rst
> b/doc/guides/rel_notes/release_19_11.rst
> index 206d287..2c51426 100644
> --- a/doc/guides/rel_notes/release_19_11.rst
> +++ b/doc/guides/rel_notes/release_19_11.rst
> @@ -193,6 +193,21 @@ New Features
> gives ability to print port supported ptypes in different protocol layers.
>
>
> +* **Add support of support dynamic fields and flags in mbuf.**
> +
> + This new feature adds the ability to dynamically register some room
> + for a field or a flag in the mbuf structure. This is typically used
> + for specific offload features, where adding a static field or flag
> + in the mbuf is not justified.
> +
> +* **Extended metadata support in rte_flow.**
> +
> + Flow metadata is extended to both Rx and Tx.
> +
> + * Tx metadata can also be set by SET_META action of rte_flow.
> + * Rx metadata is delivered to host via a dynamic field of ``rte_mbuf`` with
> + PKT_RX_DYNF_METADATA.
> +
> Removed Items
> -------------
>
> diff --git a/lib/librte_ethdev/rte_ethdev.h b/lib/librte_ethdev/rte_ethdev.h
> index 33c528b..6ad5e1b 100644
> --- a/lib/librte_ethdev/rte_ethdev.h
> +++ b/lib/librte_ethdev/rte_ethdev.h
> @@ -1048,7 +1048,6 @@ struct rte_eth_conf {
> #define DEV_RX_OFFLOAD_KEEP_CRC 0x00010000
> #define DEV_RX_OFFLOAD_SCTP_CKSUM 0x00020000
> #define DEV_RX_OFFLOAD_OUTER_UDP_CKSUM 0x00040000
> -
> #define DEV_RX_OFFLOAD_CHECKSUM (DEV_RX_OFFLOAD_IPV4_CKSUM | \
> DEV_RX_OFFLOAD_UDP_CKSUM | \
> DEV_RX_OFFLOAD_TCP_CKSUM)
> diff --git a/lib/librte_ethdev/rte_ethdev_version.map
> b/lib/librte_ethdev/rte_ethdev_version.map
> index e59d516..a5bf643 100644
> --- a/lib/librte_ethdev/rte_ethdev_version.map
> +++ b/lib/librte_ethdev/rte_ethdev_version.map
> @@ -288,4 +288,7 @@ EXPERIMENTAL {
> rte_eth_rx_burst_mode_get;
> rte_eth_tx_burst_mode_get;
> rte_eth_burst_mode_option_name;
> + rte_flow_dynf_metadata_offs;
> + rte_flow_dynf_metadata_mask;
> + rte_flow_dynf_metadata_register;
> };
> diff --git a/lib/librte_ethdev/rte_flow.c b/lib/librte_ethdev/rte_flow.c
> index ca0f680..6090177 100644
> --- a/lib/librte_ethdev/rte_flow.c
> +++ b/lib/librte_ethdev/rte_flow.c
> @@ -12,10 +12,18 @@
> #include <rte_errno.h>
> #include <rte_branch_prediction.h>
> #include <rte_string_fns.h>
> +#include <rte_mbuf.h>
> +#include <rte_mbuf_dyn.h>
> #include "rte_ethdev.h"
> #include "rte_flow_driver.h"
> #include "rte_flow.h"
>
> +/* Mbuf dynamic field name for metadata. */
> +int rte_flow_dynf_metadata_offs = -1;
> +
> +/* Mbuf dynamic field flag bit number for metadata. */
> +uint64_t rte_flow_dynf_metadata_mask;
> +
> /**
> * Flow elements description tables.
> */
> @@ -157,8 +165,41 @@ struct rte_flow_desc_data {
> MK_FLOW_ACTION(DEC_TCP_SEQ, sizeof(rte_be32_t)),
> MK_FLOW_ACTION(INC_TCP_ACK, sizeof(rte_be32_t)),
> MK_FLOW_ACTION(DEC_TCP_ACK, sizeof(rte_be32_t)),
> + MK_FLOW_ACTION(SET_META, sizeof(struct
> rte_flow_action_set_meta)),
> };
>
> +int
> +rte_flow_dynf_metadata_register(void)
> +{
> + int offset;
> + int flag;
> +
> + static const struct rte_mbuf_dynfield desc_offs = {
> + .name = MBUF_DYNF_METADATA_NAME,
> + .size = sizeof(uint32_t),
> + .align = __alignof__(uint32_t),
> + .flags = 0,
> + };
> + static const struct rte_mbuf_dynflag desc_flag = {
> + .name = MBUF_DYNF_METADATA_NAME,
> + };
> +
> + offset = rte_mbuf_dynfield_register(&desc_offs);
> + if (offset < 0)
> + goto error;
> + flag = rte_mbuf_dynflag_register(&desc_flag);
> + if (flag < 0)
> + goto error;
> + rte_flow_dynf_metadata_offs = offset;
> + rte_flow_dynf_metadata_mask = (1ULL << flag);
> + return 0;
> +
> +error:
> + rte_flow_dynf_metadata_offs = -1;
> + rte_flow_dynf_metadata_mask = 0ULL;
> + return -rte_errno;
> +}
> +
> static int
> flow_err(uint16_t port_id, int ret, struct rte_flow_error *error)
> {
> diff --git a/lib/librte_ethdev/rte_flow.h b/lib/librte_ethdev/rte_flow.h
> index 4fee105..b821557 100644
> --- a/lib/librte_ethdev/rte_flow.h
> +++ b/lib/librte_ethdev/rte_flow.h
> @@ -28,6 +28,8 @@
> #include <rte_byteorder.h>
> #include <rte_esp.h>
> #include <rte_higig.h>
> +#include <rte_mbuf.h>
> +#include <rte_mbuf_dyn.h>
>
> #ifdef __cplusplus
> extern "C" {
> @@ -418,7 +420,8 @@ enum rte_flow_item_type {
> /**
> * [META]
> *
> - * Matches a metadata value specified in mbuf metadata field.
> + * Matches a metadata value.
> + *
> * See struct rte_flow_item_meta.
> */
> RTE_FLOW_ITEM_TYPE_META,
> @@ -1263,9 +1266,17 @@ struct rte_flow_item_icmp6_nd_opt_tla_eth {
> #endif
>
> /**
> - * RTE_FLOW_ITEM_TYPE_META.
> + * @warning
> + * @b EXPERIMENTAL: this structure may change without prior notice
> *
> - * Matches a specified metadata value.
> + * RTE_FLOW_ITEM_TYPE_META
> + *
> + * Matches a specified metadata value. On egress, metadata can be set either
> by
> + * mbuf tx_metadata field with PKT_TX_METADATA flag or
> + * RTE_FLOW_ACTION_TYPE_SET_META. On ingress,
> RTE_FLOW_ACTION_TYPE_SET_META sets
> + * metadata for a packet and the metadata will be reported via mbuf
> metadata
> + * dynamic field with PKT_RX_DYNF_METADATA flag. The dynamic mbuf field
> must be
> + * registered in advance by rte_flow_dynf_metadata_register().
> */
> struct rte_flow_item_meta {
> rte_be32_t data;
> @@ -1942,6 +1953,13 @@ enum rte_flow_action_type {
> * undefined behavior.
> */
> RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK,
> +
> + /**
> + * Set metadata on ingress or egress path.
> + *
> + * See struct rte_flow_action_set_meta.
> + */
> + RTE_FLOW_ACTION_TYPE_SET_META,
> };
>
> /**
> @@ -2429,6 +2447,55 @@ struct rte_flow_action_set_mac {
> uint8_t mac_addr[RTE_ETHER_ADDR_LEN];
> };
>
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this structure may change without prior notice
> + *
> + * RTE_FLOW_ACTION_TYPE_SET_META
> + *
> + * Set metadata. Metadata set by mbuf tx_metadata field with
> + * PKT_TX_METADATA flag on egress will be overridden by this action. On
> + * ingress, the metadata will be carried by mbuf metadata dynamic field
> + * with PKT_RX_DYNF_METADATA flag if set. The dynamic mbuf field must be
> + * registered in advance by rte_flow_dynf_metadata_register().
> + *
> + * Altering partial bits is supported with mask. For bits which have never
> + * been set, unpredictable value will be seen depending on driver
> + * implementation. For loopback/hairpin packet, metadata set on Rx/Tx may
> + * or may not be propagated to the other path depending on HW capability.
> + *
> + * RTE_FLOW_ITEM_TYPE_META matches metadata.
> + */
> +struct rte_flow_action_set_meta {
> + rte_be32_t data;
> + rte_be32_t mask;
> +};
> +
> +/* Mbuf dynamic field offset for metadata. */
> +extern int rte_flow_dynf_metadata_offs;
> +
> +/* Mbuf dynamic field flag mask for metadata. */
> +extern uint64_t rte_flow_dynf_metadata_mask;
> +
> +/* Mbuf dynamic field pointer for metadata. */
> +#define RTE_FLOW_DYNF_METADATA(m) \
> + RTE_MBUF_DYNFIELD((m), rte_flow_dynf_metadata_offs, uint32_t *)
> +
> +/* Mbuf dynamic flag for metadata. */
> +#define PKT_RX_DYNF_METADATA (rte_flow_dynf_metadata_mask)
> +
> +__rte_experimental
> +static inline uint32_t
> +rte_flow_dynf_metadata_get(struct rte_mbuf *m) {
> + return *RTE_FLOW_DYNF_METADATA(m);
> +}
> +
> +__rte_experimental
> +static inline void
> +rte_flow_dynf_metadata_set(struct rte_mbuf *m, uint32_t v) {
> + *RTE_FLOW_DYNF_METADATA(m) = v;
> +}
> +
> /*
> * Definition of a single action.
> *
> @@ -2662,6 +2729,32 @@ enum rte_flow_conv_op {
> };
>
> /**
> + * Check if mbuf dynamic field for metadata is registered.
> + *
> + * @return
> + * True if registered, false otherwise.
> + */
> +__rte_experimental
> +static inline int
> +rte_flow_dynf_metadata_avail(void) {
> + return !!rte_flow_dynf_metadata_mask;
> +}
> +
> +/**
> + * Register mbuf dynamic field and flag for metadata.
> + *
> + * This function must be called prior to use SET_META action in order to
> + * register the dynamic mbuf field. Otherwise, the data cannot be delivered to
> + * application.
> + *
> + * @return
> + * 0 on success, a negative errno value otherwise and rte_errno is set.
> + */
> +__rte_experimental
> +int
> +rte_flow_dynf_metadata_register(void);
> +
> +/**
> * Check whether a flow rule can be created on a given port.
> *
> * The flow rule is validated for correctness and whether it could be accepted
> diff --git a/lib/librte_mbuf/rte_mbuf_dyn.h b/lib/librte_mbuf/rte_mbuf_dyn.h
> index 2e9d418..a4a0cf5 100644
> --- a/lib/librte_mbuf/rte_mbuf_dyn.h
> +++ b/lib/librte_mbuf/rte_mbuf_dyn.h
> @@ -234,6 +234,10 @@ int rte_mbuf_dynflag_lookup(const char *name,
> __rte_experimental
> void rte_mbuf_dyn_dump(FILE *out);
>
> -/* Placeholder for dynamic fields and flags declarations. */
> -
> +/*
> + * Placeholder for dynamic fields and flags declarations.
> + * This is centralizing point to gather all field names
> + * and parameters together.
> + */
> +#define MBUF_DYNF_METADATA_NAME "rte_flow_dynfield_metadata"
> #endif
> --
> 1.8.3.1
Thanks,
Ori
More information about the dev
mailing list