From a072f81df6492905ea15efe34400f110ba114376 Mon Sep 17 00:00:00 2001
From: Giga <52905881+giga-a@users.noreply.github.com>
Date: Tue, 14 Sep 2021 16:12:15 -0700
Subject: [PATCH] [auto-b] added stats section - ref ce1fc3d68e9b

---
 app.js           |  8 +++----
 modules/stats.js | 57 ++++++++++++++++++++++++++++++++++++++++++++----
 public/app.html  | 54 ++++++++++++++++++++++++++++++++++++---------
 3 files changed, 101 insertions(+), 18 deletions(-)

diff --git a/app.js b/app.js
index 1fc1e88..ddd7896 100755
--- a/app.js
+++ b/app.js
@@ -316,9 +316,9 @@ app.post('/analyze_string', async function (req, res, next) {
       }
 
       helper.log_to_file_queue(req.body.uuid, '[Done] Checking user profiles normal')
-      if (req.body.option.includes('CategoriesStats')) {
+      if (req.body.option.includes('CategoriesStats') || req.body.option.includes('MetadataStats')) {
         helper.log_to_file_queue(req.body.uuid, '[Starting] Generate stats')
-        stats_default = await stats.get_stats(user_info_normal.data)
+        stats_default = await stats.get_stats(req,user_info_normal.data)
         helper.log_to_file_queue(req.body.uuid, '[Done] Generate stats')
       }
     }
@@ -490,7 +490,7 @@ app.post('/analyze_string', async function (req, res, next) {
 
     helper.log_to_file_queue(req.body.uuid, '[Finished] Analyzing: ' + req.body.string + ' Task: ' + req.body.uuid)
 
-    /* fs.writeFileSync('./test.json', JSON.stringify({
+     /*fs.writeFileSync('./test.json', JSON.stringify({
       username: username,
       uuid: temp_uuid,
       info,
@@ -505,7 +505,7 @@ app.post('/analyze_string', async function (req, res, next) {
       graph: graph,
       stats: stats_default,
       logs: logs
-    }, null, 2) , 'utf-8'); */
+    }, null, 2) , 'utf-8');*/
 
     res.json({
       username: username,
diff --git a/modules/stats.js b/modules/stats.js
index fdbfd54..c933cfd 100644
--- a/modules/stats.js
+++ b/modules/stats.js
@@ -46,17 +46,66 @@ async function get_stats_by_value (data, value) {
   return temp_array
 }
 
-async function get_stats (data) {
+async function get_metadata(data, type){
+  const temp_array = []
+  try {
+    const temp_filtered = data.filter(item => item.status === type)
+    await temp_filtered.forEach(site => {
+      if ('metadata' in site) {
+        if (site.metadata !== 'unavailable' && site.metadata.length > 0) {
+          site.metadata.forEach(meta => {
+            if ('content' in meta) {
+              let temp_content = ''
+              if ('name' in meta) {
+                temp_content = meta.content
+              } else if ('itemprop' in meta) {
+                temp_content = meta.content
+              } else if ('property' in meta) {
+                temp_content = meta.content
+              }
+
+              if (temp_content != ''){
+                const entry = temp_array.find( x => x[1] === temp_content);
+                if (entry) {
+                    ++entry[0];
+                } else {
+                    temp_array.push([1,temp_content]);
+                }
+              }
+            }
+          })
+        }
+      }
+    })
+  }
+  catch(error){
+    helper.verbose && console.log(error)
+  }
+
+  if (temp_array.length >0) {
+    temp_array.sort(function(a,b) {return b[0]-a[0]})
+  }
+
+  return temp_array
+}
+
+async function get_stats (req, data) {
   let categories = {}
   let countries = {}
+  let metadata = []
   try {
-    categories = await get_stats_by_value(data, 'type')
-    countries = await get_stats_by_value(data, 'country')
+    if (req.body.option.includes('CategoriesStats')) {
+      categories = await get_stats_by_value(data, 'type')
+      countries = await get_stats_by_value(data, 'country')
+    }
+    if (req.body.option.includes('ExtractMetadata') && req.body.option.includes('MetadataStats')){
+      metadata = await get_metadata(data, 'good')
+    }
   } catch (error) {
     helper.verbose && console.log(error)
   }
 
-  return { categories: categories, countries: countries }
+  return { categories: categories, countries: countries, metadata: metadata}
 }
 
 module.exports = {
diff --git a/public/app.html b/public/app.html
index 8ee98fe..d1034fb 100755
--- a/public/app.html
+++ b/public/app.html
@@ -260,6 +260,7 @@
   #most-common-words-section,
   #words-info-section,
   #stats-section,
+  #metadata-section,
   #custom-search-section,
   #names-origins-section {
     display: none;
@@ -643,6 +644,11 @@
     text-align: right;
   }
 
+  #metadata-table tr td:nth-child(1),
+  #metadata-table tr th:nth-child(1) {
+    width: 50px;
+  }
+
   .div-right-icon {
     float: right;
     font-size: 16px;
@@ -871,6 +877,14 @@
         <div id="force-graph-canvas" class="collapse show">
         </div>
       </div>
+      <div class="box-border-style" id="metadata-section">
+        <div class="div-header">
+          <div class="div-left">Meta</div>
+          <div class="div-right"><i class="div-right-icon fa fa-caret-up"></i></div>
+        </div>
+        <div id="metadata-table" class="collapse show">
+        </div>
+      </div>
       <div class="box-border-style" id="stats-section">
         <div class="div-header">
           <div class="div-left">Stats</div>
@@ -977,6 +991,11 @@
         value: 'CategoriesStats',
         name: 'Generate stats of categories',
         disable: false,
+      }, {
+        group: 'Stats',
+        value: 'MetadataStats',
+        name: 'Generate stats of Metadata',
+        disable: false,
       },
       {
         group: 'Other options',
@@ -1178,6 +1197,8 @@
       $('#logs').html('');
       $('#stats-tables').html('');
       $('#stats-section').hide();
+      $('#metadata-table').html('');
+      $('#metadata-section').hide();
       clearInterval(interval_logs);
     }
 
@@ -1435,18 +1456,31 @@
             }
 
             Object.keys(data.stats).forEach((type_key) => {
-              if (Object.keys(data.stats[type_key]).length > 0) {
-                Object.keys(data.stats[type_key]).forEach((status_key) => {
-                  temp_tr = '';
-                  data.stats[type_key][status_key].forEach((item) => {
-                    temp_tr += `<tr><td>${item[0]}</td><td>%${item[1]}</td></tr>`;
+              if(typeof data.stats[type_key] === "object" && !Array.isArray(data.stats[type_key])){
+                if (Object.keys(data.stats[type_key]).length > 0) {
+                  Object.keys(data.stats[type_key]).forEach((status_key) => {
+                    temp_tr = '';
+                    data.stats[type_key][status_key].forEach((item) => {
+                      temp_tr += `<tr><td>${item[0]}</td><td>%${item[1]}</td></tr>`;
+                    });
+                    if (temp_tr.length > 0) {
+                      $('#stats-tables').last().append(`<table><tr><th>[${status_key} profiles] ${type_key}</th><th>Percentage</th></tr>${temp_tr}</table>`);
+                    }
                   });
-                  if (temp_tr.length > 0) {
-                    $('#stats-tables').last().append(`<table><tr><th>[${status_key} profiles] ${type_key}</th><th>Percentage</th></tr>${temp_tr}</table>`);
-                  }
+                }
+                $('#stats-section').show();
+              }
+              if(typeof data.stats[type_key] === "object" && Array.isArray(data.stats[type_key])){
+                temp_tr = '';
+                data.stats[type_key].forEach((item) => {
+                  temp_tr += `<tr><td>${item[0]}</td><td>${item[1]}</td></tr>`;
                 });
 
-                $('#stats-section').show();
+                if (temp_tr.length > 0) {
+                  $('#metadata-table').last().append(`<table><tr><th>Total</th><th>Metadata string</th></tr>${temp_tr}</table>`);
+                }
+
+                $('#metadata-section').show();
               }
             });
 
@@ -1591,7 +1625,7 @@
       e.preventDefault();
       $('#analyzed-options')[0].selectize.clear();
       $('#analyzed-options')[0].selectize.setValue(['WordInfo', 'MostCommon', 'FindOrigins', 'LookUps', 'CustomSearch', 'FindUserProfilesFast,GetUserProfilesFast', 'ExtractMetadata', 'ExtractPatterns', 'NetworkGraph', 'FindNumbers',
-        'CategoriesStats',
+        'CategoriesStats','MetadataStats',
         'FindSymbols', 'SplitWordsByAlphabet', 'SplitWordsByUpperCase',
       ]);
     });