<html xmlns:o="urn:schemas-microsoft-com:office:office" xmlns:w="urn:schemas-microsoft-com:office:word" xmlns:dt="uuid:C2F41010-65B3-11d1-A29F-00AA00C14882" xmlns:m="http://schemas.microsoft.com/office/2004/12/omml" xmlns="http://www.w3.org/TR/REC-html40">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<meta name="Generator" content="Microsoft Word 15 (filtered medium)">
<style><!--
/* Font Definitions */
@font-face
{font-family:"Cambria Math";
panose-1:2 4 5 3 5 4 6 3 2 4;}
@font-face
{font-family:Aptos;
panose-1:2 11 0 4 2 2 2 2 2 4;}
@font-face
{font-family:Consolas;
panose-1:2 11 6 9 2 2 4 3 2 4;}
/* Style Definitions */
p.MsoNormal, li.MsoNormal, div.MsoNormal
{margin:0cm;
font-size:12.0pt;
font-family:"Aptos",sans-serif;}
a:link, span.MsoHyperlink
{mso-style-priority:99;
color:blue;
text-decoration:underline;}
pre
{mso-style-priority:99;
mso-style-link:"HTML Preformatted Char";
margin:0cm;
margin-bottom:.0001pt;
font-size:10.0pt;
font-family:"Courier New";}
span.HTMLPreformattedChar
{mso-style-name:"HTML Preformatted Char";
mso-style-priority:99;
mso-style-link:"HTML Preformatted";
font-family:Consolas;}
span.EmailStyle20
{mso-style-type:personal-reply;
font-family:"Aptos",sans-serif;
color:windowtext;}
.MsoChpDefault
{mso-style-type:export-only;
font-size:10.0pt;
mso-ligatures:none;}
@page WordSection1
{size:612.0pt 792.0pt;
margin:72.0pt 72.0pt 72.0pt 72.0pt;}
div.WordSection1
{page:WordSection1;}
--></style>
</head>
<body lang="EN-IN" link="blue" vlink="purple" style="word-wrap:break-word">
<div class="WordSection1">
<p class="MsoNormal"><span style="mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<p class="MsoNormal">Get <a href="https://aka.ms/GetOutlookForMac">Outlook for Mac
</a><span style="mso-fareast-language:EN-US"><o:p></o:p></span></p>
<p class="MsoNormal"><span style="mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<div id="mail-editor-reference-message-container">
<div>
<div>
<div style="border:none;border-top:solid #B5C4DF 1.0pt;padding:3.0pt 0cm 0cm 0cm">
<p class="MsoNormal" style="margin-bottom:12.0pt"><b><span style="color:black">From:
</span></b><span style="color:black">David Marchand <david.marchand@redhat.com><br>
<b>Date: </b>Tuesday, 8 July 2025 at 6:01</span><span style="font-family:"Arial",sans-serif;color:black"> </span><span style="color:black">PM<br>
<b>To: </b>dev@dpdk.org <dev@dpdk.org><br>
<b>Cc: </b>stable@dpdk.org <stable@dpdk.org>, Jerin Jacob <jerinj@marvell.com>, Kiran Kumar Kokkilagadda <kirankumark@marvell.com>, Nithin Kumar Dabilpuram <ndabilpuram@marvell.com>, Zhirun Yan <yanzhirun_163@163.com>, Pavan Nikhilesh Bhagavatula <pbhagavatula@marvell.com><br>
<b>Subject: </b>[EXTERNAL] [PATCH v3 14/18] graph: fix unaligned access in stats<o:p></o:p></span></p>
</div>
<div>
<p class="MsoNormal" style="mso-line-height-alt:.75pt"><span style="font-size:1.0pt;color:white">UBSan reports: ..</span><span style="font-size:1.0pt;font-family:"Arial",sans-serif;color:white"> </span><span style="font-size:1.0pt;color:white">/lib/graph/graph_stats.</span><span style="font-size:1.0pt;font-family:"Arial",sans-serif;color:white"> </span><span style="font-size:1.0pt;color:white">c:</span><span style="font-size:1.0pt;font-family:"Arial",sans-serif;color:white"> </span><span style="font-size:1.0pt;color:white">208:</span><span style="font-size:1.0pt;font-family:"Arial",sans-serif;color:white"> </span><span style="font-size:1.0pt;color:white">13:
runtime error: member access within misaligned address 0x000054742c50 for type 'struct rte_graph_cluster_stats', which requires 64 byte alignment ..</span><span style="font-size:1.0pt;font-family:"Arial",sans-serif;color:white"> </span><span style="font-size:1.0pt;color:white">/lib/graph/graph_stats.</span><span style="font-size:1.0pt;font-family:"Arial",sans-serif;color:white"> </span><span style="font-size:1.0pt;color:white">c:</span><span style="font-size:1.0pt;font-family:"Arial",sans-serif;color:white"> </span><span style="font-size:1.0pt;color:white">257:</span><span style="font-size:1.0pt;font-family:"Arial",sans-serif;color:white"> </span><span style="font-size:1.0pt;color:white">12:
runtime error:</span><span style="font-size:1.0pt;font-family:"Arial",sans-serif;color:white"> </span><span style="font-size:1.0pt;color:white"><o:p></o:p></span></p>
</div>
<div>
<p class="MsoNormal" style="mso-line-height-alt:.75pt"><span style="font-size:1.0pt;color:white">ZjQcmQRYFpfptBannerStart<o:p></o:p></span></p>
</div>
<div style="border:none;border-top:solid #8D8C8C 3.0pt;padding:0cm 0cm 0cm 0cm;display:block!important;text-align:left!important;margin:0px!important;padding:16px!important;border-radius:4px!important;min-width:200px!important;background-color:#D2D0D0!important;border-top:#8d8c8c!important" id="pfptBanneraa2t9u4">
<div id="pfptBanneraa2t9u4">
<div id="pfptBanneraa2t9u4">
<p class="MsoNormal" style="line-height:13.5pt;background:#D2D0D0"><b><span lang="EN" style="font-family:"Arial",sans-serif;color:black">Prioritize security for external emails:
<o:p></o:p></span></b></p>
</div>
<div id="pfptBanneraa2t9u4">
<p class="MsoNormal" style="line-height:13.5pt;background:#D2D0D0"><span lang="EN" style="font-family:"Arial",sans-serif;color:black">Confirm sender and content safety before clicking links or opening attachments
<o:p></o:p></span></p>
</div>
</div>
<div id="pfptBanneraa2t9u4">
<p class="MsoNormal" style="background:#D2D0D0"><span lang="EN" style="color:black"><a href="https://us-phishalarm-ewt.proofpoint.com/EWT/v1/CRVmXkqW!to3Z1f8UAnU69Q-c-ZxaLqw4Hrf0nRhoZPfYuYT9mJ7YnAXilGrf--fL-Zm7l1VQHd3bjUXwIMnvcu_suMpHO0hzN2TUxcSNBPH6eYM8qA$"><span lang="EN-GB" style="text-decoration:none"><o:p></o:p></span></a></span></p>
<div style="border:solid #666666 1.0pt;padding:0cm 0cm 0cm 0cm;display:inline-block!important;display:inline-block;visibility:visible!important;opacity:1!important;color:#000000!important;font-size:14px!important;font-weight:normal!important;text-decoration:none!important;border-radius:2px!important;padding:16px!important;margin:16px!important;white-space:nowrap!important;width:fit-content!important">
<p class="MsoNormal" style="background:#D2D0D0"><span lang="EN" style="font-family:"Arial",sans-serif;color:black"><a href="https://us-phishalarm-ewt.proofpoint.com/EWT/v1/CRVmXkqW!to3Z1f8UAnU69Q-c-ZxaLqw4Hrf0nRhoZPfYuYT9mJ7YnAXilGrf--fL-Zm7l1VQHd3bjUXwIMnvcu_suMpHO0hzN2TUxcSNBPH6eYM8qA$"><span style="color:black;text-decoration:none">Report Suspicious
</span><span lang="EN-GB" style="color:black;text-decoration:none"><o:p></o:p></span></a></span></p>
</div>
<p class="MsoNormal" style="background:#D2D0D0"><span lang="EN"><o:p> </o:p></span></p>
</div>
<div>
<p class="MsoNormal" style="background:#D2D0D0"><span lang="EN" style="color:black"> </span><span lang="EN"><o:p></o:p></span></p>
</div>
</div>
<div>
<p class="MsoNormal" style="mso-line-height-alt:.75pt"><span style="font-size:1.0pt;color:white">ZjQcmQRYFpfptBannerEnd<o:p></o:p></span></p>
</div>
<pre style="white-space:pre-wrap"><span style="font-size:12.0pt;font-family:"Arial",sans-serif">UBSan reports:<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"><o:p> </o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> ../lib/graph/graph_stats.c:208:13: runtime error:<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> member access within misaligned address 0x000054742c50<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> for type 'struct rte_graph_cluster_stats',<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> which requires 64 byte alignment<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"><o:p> </o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> ../lib/graph/graph_stats.c:257:12: runtime error:<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> member access within misaligned address 0x00002219fd30<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> for type 'struct rte_graph_cluster_stats',<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> which requires 64 byte alignment<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"><o:p> </o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">The current code goes into various complex (non aligned) reallocations /<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">memset / memcpy.<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"><o:p> </o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">Simplify this by computing how many nodes are present in the<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">cluster of graphes.<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">Then directly call rte_malloc for the whole stats object.<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"><o:p> </o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">As a bonus, this change also fixes leaks:<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">- if any error occurred before call to rte_malloc, since the xstats<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> objects stored in the glibc allocated stats object were not freed,<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">- if an allocation failure occurs, with constructs using<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> ptr = realloc(ptr, sz), since the original ptr is lost,<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"><o:p> </o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">Fixes: af1ae8b6a32c ("graph: implement stats")<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">Cc: stable@dpdk.org<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"><o:p> </o:p></span></pre>
<div style="mso-element:para-border-div;border:none;border-bottom:solid windowtext 1.0pt;padding:0cm 0cm 1.0pt 0cm">
<pre style="border:none;padding:0cm"><span style="font-size:12.0pt;font-family:"Arial",sans-serif">Signed-off-by: David Marchand <david.marchand@redhat.com><o:p></o:p></span></pre>
</div>
<pre><span lang="EN-US" style="font-size:11.0pt;font-family:Consolas"><o:p> </o:p></span></pre>
<pre><span lang="EN-US" style="font-size:12.0pt;font-family:"Aptos",sans-serif"><o:p> </o:p></span></pre>
<pre><span lang="EN-US" style="font-size:11.0pt;font-family:Consolas">Acked-by: Kiran Kumar Kokkilagadda <<a href="mailto:kirankumark@marvell.com">kirankumark@marvell.com</a>></span><span style="font-size:12.0pt;font-family:"Arial",sans-serif"><o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Aptos",sans-serif"><o:p> </o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> lib/graph/graph_stats.c | 102 +++++++++++++++++++++++-----------------<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> 1 file changed, 58 insertions(+), 44 deletions(-)<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"><o:p> </o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">diff --git a/lib/graph/graph_stats.c b/lib/graph/graph_stats.c<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">index 583ad8dbd5..e9b183bf13 100644<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">--- a/lib/graph/graph_stats.c<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+++ b/lib/graph/graph_stats.c<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">@@ -37,7 +37,6 @@ struct __rte_cache_aligned rte_graph_cluster_stats {<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> int socket_id;<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> bool dispatch;<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> void *cookie;<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">- size_t sz;<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> <o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> struct cluster_node clusters[];<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> };<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">@@ -178,15 +177,55 @@ graph_cluster_stats_cb_dispatch(bool is_first, bool is_last, void *cookie,<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> return graph_cluster_stats_cb(true, is_first, is_last, cookie, stat);<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> };<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> <o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+static uint32_t<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+cluster_count_nodes(const struct cluster *cluster)<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+{<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+ rte_node_t *nodes = NULL;<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+ uint32_t max_nodes = 0;<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+ for (unsigned int i = 0; i < cluster->nb_graphs; i++) {<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+ struct graph_node *graph_node;<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+ STAILQ_FOREACH(graph_node, &cluster->graphs[i]->node_list, next) {<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+ unsigned int n;<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+ for (n = 0; n < max_nodes; n++) {<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+ if (nodes[n] != graph_node->node->id)<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+ continue;<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+ break;<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+ }<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+ if (n == max_nodes) {<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+ rte_node_t *new_nodes;<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+ max_nodes++;<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+ new_nodes = realloc(nodes, max_nodes * sizeof(nodes[0]));<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+ if (new_nodes == NULL) {<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+ free(nodes);<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+ return 0;<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+ }<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+ nodes = new_nodes;<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+ nodes[n] = graph_node->node->id;<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+ }<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+ }<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+ }<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+ free(nodes);<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+ return max_nodes;<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+}<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> static struct rte_graph_cluster_stats *<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> stats_mem_init(struct cluster *cluster,<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> const struct rte_graph_cluster_stats_param *prm)<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> {<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">- size_t sz = sizeof(struct rte_graph_cluster_stats);<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> struct rte_graph_cluster_stats *stats;<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> rte_graph_cluster_stats_cb_t fn;<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> int socket_id = prm->socket_id;<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> uint32_t cluster_node_size;<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+ uint32_t max_nodes;<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+ max_nodes = cluster_count_nodes(cluster);<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+ if (max_nodes == 0)<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+ return NULL;<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> <o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> /* Fix up callback */<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> fn = prm->fn;<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">@@ -203,25 +242,23 @@ stats_mem_init(struct cluster *cluster,<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> cluster_node_size += cluster->nb_graphs * sizeof(struct rte_node *);<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> cluster_node_size = RTE_ALIGN(cluster_node_size, RTE_CACHE_LINE_SIZE);<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> <o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">- stats = realloc(NULL, sz);<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+ stats = rte_zmalloc_socket(NULL, sizeof(struct rte_graph_cluster_stats) +<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+ max_nodes * cluster_node_size, 0, socket_id);<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> if (stats) {<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">- memset(stats, 0, sz);<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> stats->fn = fn;<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> stats->cluster_node_size = cluster_node_size;<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> stats->max_nodes = 0;<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> stats->socket_id = socket_id;<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> stats->cookie = prm->cookie;<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">- stats->sz = sz;<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> }<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> <o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> return stats;<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> }<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> <o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> static int<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">-stats_mem_populate(struct rte_graph_cluster_stats **stats_in,<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+stats_mem_populate(struct rte_graph_cluster_stats *stats,<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> struct rte_graph *graph, struct graph_node *graph_node)<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> {<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">- struct rte_graph_cluster_stats *stats = *stats_in;<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> rte_node_t id = graph_node->node->id;<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> struct cluster_node *cluster;<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> struct rte_node *node;<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">@@ -247,21 +284,12 @@ stats_mem_populate(struct rte_graph_cluster_stats **stats_in,<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> cluster = RTE_PTR_ADD(cluster, stats->cluster_node_size);<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> }<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> <o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">- /* Hey, it is a new node, allocate space for it in the reel */<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">- stats = realloc(stats, stats->sz + stats->cluster_node_size);<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">- if (stats == NULL)<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">- SET_ERR_JMP(ENOMEM, err, "Realloc failed");<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">- *stats_in = NULL;<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">-<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">- /* Clear the new struct cluster_node area */<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">- cluster = RTE_PTR_ADD(stats, stats->sz),<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">- memset(cluster, 0, stats->cluster_node_size);<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> memcpy(cluster->stat.name, graph_node->node->name, RTE_NODE_NAMESIZE);<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> cluster->stat.id = graph_node->node->id;<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> cluster->stat.hz = rte_get_timer_hz();<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> node = graph_node_id_to_ptr(graph, id);<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> if (node == NULL)<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">- SET_ERR_JMP(ENOENT, free, "Failed to find node %s in graph %s",<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+ SET_ERR_JMP(ENOENT, err, "Failed to find node %s in graph %s",<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> graph_node->node->name, graph->name);<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> cluster->nodes[cluster->nb_nodes++] = node;<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> if (graph_node->node->xstats) {<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">@@ -270,7 +298,7 @@ stats_mem_populate(struct rte_graph_cluster_stats **stats_in,<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> sizeof(uint64_t) * graph_node->node->xstats->nb_xstats,<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> RTE_CACHE_LINE_SIZE, stats->socket_id);<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> if (cluster->stat.xstat_count == NULL)<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">- SET_ERR_JMP(ENOMEM, free, "Failed to allocate memory node %s graph %s",<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+ SET_ERR_JMP(ENOMEM, err, "Failed to allocate memory node %s graph %s",<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> graph_node->node->name, graph->name);<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> <o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> cluster->stat.xstat_desc = rte_zmalloc_socket(NULL,<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">@@ -278,7 +306,7 @@ stats_mem_populate(struct rte_graph_cluster_stats **stats_in,<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> RTE_CACHE_LINE_SIZE, stats->socket_id);<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> if (cluster->stat.xstat_desc == NULL) {<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> rte_free(cluster->stat.xstat_count);<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">- SET_ERR_JMP(ENOMEM, free, "Failed to allocate memory node %s graph %s",<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+ SET_ERR_JMP(ENOMEM, err, "Failed to allocate memory node %s graph %s",<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> graph_node->node->name, graph->name);<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> }<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> <o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">@@ -288,30 +316,20 @@ stats_mem_populate(struct rte_graph_cluster_stats **stats_in,<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> RTE_NODE_XSTAT_DESC_SIZE) < 0) {<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> rte_free(cluster->stat.xstat_count);<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> rte_free(cluster->stat.xstat_desc);<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">- SET_ERR_JMP(E2BIG, free,<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+ SET_ERR_JMP(E2BIG, err,<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> "Error description overflow node %s graph %s",<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> graph_node->node->name, graph->name);<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> }<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> }<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> }<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> <o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">- stats->sz += stats->cluster_node_size;<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> stats->max_nodes++;<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">- *stats_in = stats;<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> <o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> return 0;<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">-free:<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">- free(stats);<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> err:<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> return -rte_errno;<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> }<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> <o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">-static void<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">-stats_mem_fini(struct rte_graph_cluster_stats *stats)<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">-{<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">- free(stats);<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">-}<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">-<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> static void<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> cluster_init(struct cluster *cluster)<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> {<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">@@ -381,10 +399,7 @@ struct rte_graph_cluster_stats *<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> rte_graph_cluster_stats_create(const struct rte_graph_cluster_stats_param *prm)<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> {<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> struct rte_graph_cluster_stats *stats, *rc = NULL;<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">- struct graph_node *graph_node;<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> struct cluster cluster;<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">- struct graph *graph;<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">- const char *pattern;<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> rte_graph_t i;<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> <o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> /* Sanity checks */<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">@@ -402,37 +417,36 @@ rte_graph_cluster_stats_create(const struct rte_graph_cluster_stats_param *prm)<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> graph_spinlock_lock();<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> /* Expand graph pattern and add the graph to the cluster */<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> for (i = 0; i < prm->nb_graph_patterns; i++) {<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">- pattern = prm->graph_patterns[i];<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">- if (expand_pattern_to_cluster(&cluster, pattern))<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+ if (expand_pattern_to_cluster(&cluster, prm->graph_patterns[i]))<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> goto bad_pattern;<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> }<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> <o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> /* Alloc the stats memory */<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> stats = stats_mem_init(&cluster, prm);<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> if (stats == NULL)<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">- SET_ERR_JMP(ENOMEM, bad_pattern, "Failed alloc stats memory");<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+ SET_ERR_JMP(ENOMEM, bad_pattern, "Failed rte_malloc for stats memory");<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> <o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> /* Iterate over M(Graph) x N (Nodes in graph) */<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> for (i = 0; i < cluster.nb_graphs; i++) {<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+ struct graph_node *graph_node;<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+ struct graph *graph;<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> graph = cluster.graphs[i];<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> STAILQ_FOREACH(graph_node, &graph->node_list, next) {<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> struct rte_graph *graph_fp = graph->graph;<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">- if (stats_mem_populate(&stats, graph_fp, graph_node))<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+ if (stats_mem_populate(stats, graph_fp, graph_node))<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> goto realloc_fail;<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> }<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> if (graph->graph->model == RTE_GRAPH_MODEL_MCORE_DISPATCH)<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> stats->dispatch = true;<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> }<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> <o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">- /* Finally copy to hugepage memory to avoid pressure on rte_realloc */<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">- rc = rte_malloc_socket(NULL, stats->sz, 0, stats->socket_id);<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">- if (rc)<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">- rte_memcpy(rc, stats, stats->sz);<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">- else<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">- SET_ERR_JMP(ENOMEM, realloc_fail, "rte_malloc failed");<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+ rc = stats;<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+ stats = NULL;<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> <o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> realloc_fail:<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">- stats_mem_fini(stats);<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+ if (stats != NULL)<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">+ rte_graph_cluster_stats_destroy(stats);<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> bad_pattern:<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> graph_spinlock_unlock();<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"> cluster_fini(&cluster);<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">-- <o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif">2.50.0<o:p></o:p></span></pre>
<pre><span style="font-size:12.0pt;font-family:"Arial",sans-serif"><o:p> </o:p></span></pre>
</div>
</div>
</div>
</div>
</body>
</html>