From e2d9c5d10114429e08d2eaa624952ae880e5d149 Mon Sep 17 00:00:00 2001 From: Giga <52905881+giga-a@users.noreply.github.com> Date: Wed, 29 Jun 2022 12:07:16 -0700 Subject: [PATCH] Add files via upload --- README.md | 1 + app.js | 44 +++++++++++++++++++++----------------- modules/engine.js | 9 +++----- modules/external-apis.js | 6 +++--- modules/extraction.js | 8 ++++--- modules/fast-scan.js | 24 ++++++++++----------- modules/helper.js | 44 +++++++++++++++++++++----------------- modules/name-analysis.js | 7 +++--- modules/slow-scan.js | 23 +++++++++----------- modules/special-scan.js | 13 ++++------- modules/stats.js | 4 ++-- modules/string-analysis.js | 11 ++++------ modules/visualize.js | 8 +++---- package-lock.json | 20 +++++++++++++++++ package.json | 4 +++- public/graph.html | 42 ++++++++++++++++++++++++++++++++++++ 16 files changed, 165 insertions(+), 103 deletions(-) create mode 100644 public/graph.html diff --git a/README.md b/README.md index c3f78e2..3b37224 100755 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ The analysis and public extracted information from this OSINT tool could help in `This project is currently used by some law enforcement agencies in countries where resources are limited` +- [New] Works on nodejs v18.4.0 - [New] Meta nested module is queued for testing - [New] Updates queued for testing - [Info] Feel free to fork and contribute diff --git a/app.js b/app.js index 40278fa..3da218c 100755 --- a/app.js +++ b/app.js @@ -7,9 +7,11 @@ // ------------------------------------------------------------- // contributors list qeeqbox/social-analyzer/graphs/contributors // ------------------------------------------------------------- - -const argv = require('yargs') - .usage('Usage: $0 --username "johndoe" --websites "youtube tiktok"\nUsage: $0 "fast" --username "johndoe"') +"use strict"; +import yargs from 'yargs' +import { hideBin } from 'yargs/helpers' +const yarg_ = yargs(hideBin(process.argv)) +const argv = yarg_.usage('Usage: $0 --username "johndoe" --websites "youtube tiktok"\nUsage: $0 "fast" --username "johndoe"') .describe('gui', 'Reserved for a gui') .default('gui', false) .boolean('gui') @@ -62,7 +64,8 @@ if (argv.output !== 'json') { console.log('[init] Detections are updated very often, make sure to get the most up-to-date ones') } -const semver = require('semver') +import semver from 'semver' + if (semver.satisfies(process.version, '>13 || <13')) { if (argv.output !== 'json') { console.log('[init] NodeJS Version Check') @@ -74,29 +77,30 @@ if (semver.satisfies(process.version, '>13 || <13')) { process.exit(1) } -const express = require('express') -const fs = require('fs') -const tokenizer = require('wink-tokenizer') -const generatorics = require('generatorics') -const HttpsProxyAgent = require('https-proxy-agent') -const PrettyError = require('pretty-error') +import express from 'express' +import fs from 'fs' +import tokenizer from 'wink-tokenizer' +import generatorics from 'generatorics' +import HttpsProxyAgent from 'https-proxy-agent' +import PrettyError from 'pretty-error' + const pe = new PrettyError() -require('express-async-errors') +import 'express-async-errors' //const _tokenizer = tokenizer() if (!fs.existsSync('logs')) { fs.mkdirSync('logs') } -const helper = require('./modules/helper.js') -const fastScan = require('./modules/fast-scan.js') -const slowScan = require('./modules/slow-scan.js') -const specialScan = require('./modules/special-scan.js') -const externalApis = require('./modules/external-apis.js') -const stringAnalysis = require('./modules/string-analysis.js') -const nameAnalysis = require('./modules/name-analysis.js') -const visualize = require('./modules/visualize.js') -const stats = require('./modules/stats.js') +import helper from './modules/helper.js' +import fastScan from './modules/fast-scan.js' +import slowScan from './modules/slow-scan.js' +import specialScan from './modules/special-scan.js' +import externalApis from './modules/external-apis.js' +import stringAnalysis from './modules/string-analysis.js' +import nameAnalysis from './modules/name-analysis.js' +import visualize from './modules/visualize.js' +import stats from './modules/stats.js' const app = express() app.set('etag', false) diff --git a/modules/engine.js b/modules/engine.js index 778d27d..9d7a10a 100755 --- a/modules/engine.js +++ b/modules/engine.js @@ -1,7 +1,5 @@ -const helper = require('./helper.js') -const { - createWorker -} = require('tesseract.js') +import helper from './helper.js' +import createWorker from 'tesseract.js' function merge_dicts (temp_dict) { const result = {} @@ -39,7 +37,6 @@ async function detect (type, uuid, username, options, site, source = '', text_on temp_profile.push(val1) temp_detected.push(val2) detections_count += val3 - // console.log(temp_profile,merge_dicts(temp_detected),detections_count) return [merge_dicts(temp_profile), merge_dicts(temp_detected), detections_count] } @@ -130,6 +127,6 @@ async function detect_logic (type, uuid, username, options, site, source = '', t return [temp_profile, temp_detected, detections_count] } -module.exports = { +export default{ detect } diff --git a/modules/external-apis.js b/modules/external-apis.js index 8f710bc..a5a829a 100755 --- a/modules/external-apis.js +++ b/modules/external-apis.js @@ -1,5 +1,5 @@ -const helper = require('./helper.js') -const async = require('async') +import helper from './helper.js' +import async from 'async' async function get_words_info (all_words, words_info) { const temp_added = [] @@ -145,7 +145,7 @@ async function custom_search_ouputs_website (uuid, name, key) { }) } -module.exports = { +export default{ check_engines, get_words_info, custom_search_ouputs diff --git a/modules/extraction.js b/modules/extraction.js index 6a3ee39..2e5f1ae 100755 --- a/modules/extraction.js +++ b/modules/extraction.js @@ -1,5 +1,7 @@ -const helper = require('./helper.js') -const cheerio = require('cheerio') +import helper from './helper.js' +import cheerio from 'cheerio' + + const strings_meta = new RegExp('regionsAllowed|width|height|color|rgba\\(|charset|viewport|refresh|equiv', 'i') async function extract_metadata (site, source) { try { @@ -84,7 +86,7 @@ async function extract_patterns (site, source) { } } -module.exports = { +export default{ extract_patterns, extract_metadata } diff --git a/modules/fast-scan.js b/modules/fast-scan.js index 7cdd1fd..2087e0c 100755 --- a/modules/fast-scan.js +++ b/modules/fast-scan.js @@ -1,12 +1,10 @@ -const helper = require('./helper.js') -const extraction = require('./extraction.js') -const async = require('async') -const sanitizeHtml = require('sanitize-html') -const { - htmlToText -} = require('html-to-text') -const cheerio = require('cheerio') -const engine = require('./engine.js') +import helper from './helper.js' +import extraction from './extraction.js' +import async from 'async' +import sanitizeHtml from 'sanitize-html' +import {convert} from 'html-to-text' +import cheerio from 'cheerio' +import engine from './engine.js' async function find_username_normal (req) { helper.log_to_file_queue(req.body.uuid, '[init] Selected websites: ' + helper.websites_entries.filter((item) => item.selected === 'true').length + ' for username: ' + req.body.string) @@ -87,7 +85,7 @@ async function find_username_site (uuid, username, options, site) { if (!options.includes('json')) { helper.log_to_file_queue(uuid, '[Checking] ' + helper.get_site_from_url(site.url)) } - const source = await helper.get_url_wrapper_text(site.url.replace('{username}', username)) + const [ret, source] = await helper.get_url_wrapper_text(site.url.replace('{username}', username)) if (source !== 'error-get-url') { let title = 'unavailable' let language = 'unavailable' @@ -99,7 +97,8 @@ async function find_username_site (uuid, username, options, site) { if (temp_profile.found >= helper.detection_level[helper.detection_level.current].found && detections_count >= helper.detection_level[helper.detection_level.current].count) { temp_profile.good = 'true' } - temp_profile.text = sanitizeHtml(htmlToText(source, { + + temp_profile.text = sanitizeHtml(convert(source, { wordwrap: false, hideLinkHrefIfSameAsText: true, ignoreHref: true, @@ -109,6 +108,7 @@ async function find_username_site (uuid, username, options, site) { temp_profile.text = 'unavailable' } + try { const $ = cheerio.load(source) title = sanitizeHtml($('title').text()) @@ -210,6 +210,6 @@ async function find_username_site (uuid, username, options, site) { }) } -module.exports = { +export default { find_username_normal } diff --git a/modules/helper.js b/modules/helper.js index 0d1e339..0077904 100755 --- a/modules/helper.js +++ b/modules/helper.js @@ -54,28 +54,31 @@ const detected_websites = { const header_options = { headers: { - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:100.0) Gecko/20100101 Firefox/100.0' + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:102.0) Gecko/20100101 Firefox/102.0' } } -const https = require('follow-redirects').https -const fs = require('fs') -const url = require('url') -const franc = require('franc') -const langs = require('langs') -const cheerio = require('cheerio') -const path = require('path') -const slash = require('slash') -const colors = require('colors/safe') -const ixora = require('ixora').QBIxora +import https from 'follow-redirects' +import fs from 'fs' +import url from 'url' +import {franc} from 'franc' +import langs from 'langs' +import cheerio from 'cheerio' +import path from 'path' +import slash from 'slash' +import colors from 'colors/safe.js' +import {QBIxora} from 'ixora' +import {fileURLToPath} from 'url'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); const sites_json_path = slash(path.join(__dirname, '..', 'data', 'sites.json')) const names_json_path = slash(path.join(__dirname, '..', 'data', 'names.json')) const dict_json_path = slash(path.join(__dirname, '..', 'data', 'dict.json')) const countries_json_path = slash(path.join(__dirname, '..', 'data', 'names.json')) const public_graph_path = slash(path.join(__dirname, '..', 'public', 'graph.html')) -let temp_ixora = new ixora('Social-Analyzer', false) +let temp_ixora = new QBIxora('Social-Analyzer', false) temp_ixora.save_base_html(public_graph_path) temp_ixora = null @@ -189,7 +192,7 @@ function get_site_from_url (_url) { async function get_url_wrapper_json (url, time = 2) { try { const http_promise = new Promise((resolve, reject) => { - const request = https.get(url, header_options, function (res) { + const request = https.https.get(url, header_options, function (res) { let body = '' res.on('data', function (chunk) { body += chunk @@ -227,15 +230,16 @@ async function get_url_wrapper_json (url, time = 2) { async function get_url_wrapper_text (url, time = 2) { const response_body = 'error-get-url' + const ret = 500 try { const http_promise = new Promise((resolve, reject) => { - const request = https.get(url, header_options, function (res) { + const request = https.https.get(url, header_options, function (res) { let body = '' res.on('data', function (chunk) { body += chunk }) res.on('end', function () { - resolve(body) + resolve([res.statusCode,body]) }) }) const timeout = (time !== 0) ? time * 1000 : 5000 @@ -256,11 +260,11 @@ async function get_url_wrapper_text (url, time = 2) { }) }) }) - const response_body = await http_promise - return response_body + const [ret, response_body] = await http_promise + return [ret, response_body] } catch (err) { verbose && console.log(err) - return response_body + return [ret, response_body] } } @@ -300,7 +304,7 @@ async function setup_tecert () { if (!fs.existsSync('eng.traineddata')) { const file = fs.createWriteStream('eng.traineddata') const http_promise = new Promise((resolve, reject) => { - const request = https.get('https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/eng.traineddata', function (response) { + const request = https.https.get('https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/eng.traineddata', function (response) { response.pipe(file) resolve(1) request.setTimeout(12000, function () { @@ -328,7 +332,7 @@ async function setup_tecert () { } } -module.exports = { +export default { strings_pages, strings_titles, top_websites, diff --git a/modules/name-analysis.js b/modules/name-analysis.js index 42ecfdb..b2fd365 100755 --- a/modules/name-analysis.js +++ b/modules/name-analysis.js @@ -1,5 +1,6 @@ -const helper = require('./helper.js') -const stringSimilarity = require('string-similarity') +import helper from './helper.js' +import stringSimilarity from 'string-similarity' + async function find_origins (req) { const found = [] for (const key in helper.parsed_names_origins) { @@ -73,6 +74,6 @@ async function find_origins (req) { return found } -module.exports = { +export default{ find_origins } diff --git a/modules/slow-scan.js b/modules/slow-scan.js index 6ef7c23..53fdc18 100755 --- a/modules/slow-scan.js +++ b/modules/slow-scan.js @@ -1,15 +1,12 @@ -const helper = require('./helper.js') -const extraction = require('./extraction.js') -const async = require('async') -const sanitizeHtml = require('sanitize-html') -const firefox = require('selenium-webdriver/firefox') -const { - Builder, - By, - Key -} = require('selenium-webdriver') -const path = require('path') -const engine = require('./engine.js') + +import helper from './helper.js' +import extraction from './extraction.js' +import async from 'async' +import sanitizeHtml from 'sanitize-html' +import firefox from 'selenium-webdriver/firefox.js' +import {Builder,By,Key} from 'selenium-webdriver' +import path from 'path' +import engine from './engine.js' if (process.platform === 'win32') { const package_path = path.join(path.dirname(require.resolve('geckodriver')), '..') @@ -194,6 +191,6 @@ async function find_username_site (uuid, username, options, site) { }) } -module.exports = { +export default{ find_username_advanced } diff --git a/modules/special-scan.js b/modules/special-scan.js index c62583a..579becf 100755 --- a/modules/special-scan.js +++ b/modules/special-scan.js @@ -1,11 +1,6 @@ -const helper = require('./helper.js') -const async = require('async') -const firefox = require('selenium-webdriver/firefox') -const { - Builder, - By, - Key -} = require('selenium-webdriver') +import helper from './helper.js' +import async from 'async' +import {Builder,By,Key} from 'selenium-webdriver' async function find_username_special (req) { const time = new Date() @@ -192,6 +187,6 @@ async function find_username_site_special_google_1 (uuid, username, site) { }) } -module.exports = { +export default{ find_username_special } diff --git a/modules/stats.js b/modules/stats.js index c933cfd..79efbe4 100644 --- a/modules/stats.js +++ b/modules/stats.js @@ -1,4 +1,4 @@ -const helper = require('./helper.js') +import helper from './helper.js' function group_by_value (list, key) { return list.reduce(function (x, y) { @@ -108,6 +108,6 @@ async function get_stats (req, data) { return { categories: categories, countries: countries, metadata: metadata} } -module.exports = { +export default{ get_stats } diff --git a/modules/string-analysis.js b/modules/string-analysis.js index a6f62bd..166ab11 100755 --- a/modules/string-analysis.js +++ b/modules/string-analysis.js @@ -1,9 +1,6 @@ -const helper = require('./helper.js') -const { - findWord -} = require('most-common-words-by-language') - -const WordsNinjaPack = require('wordsninja') +import helper from './helper.js' +import {findWord} from 'most-common-words-by-language' +import WordsNinjaPack from 'wordsninja' const WordsNinja = new WordsNinjaPack() function most_common1 (all_words, temp_words) { @@ -272,7 +269,7 @@ async function guess_age_from_string(req) { return results } -module.exports = { +export default{ get_maybe_words, find_symbols, find_numbers, diff --git a/modules/visualize.js b/modules/visualize.js index 0979ed3..34b4448 100755 --- a/modules/visualize.js +++ b/modules/visualize.js @@ -1,9 +1,9 @@ -const ixora = require('ixora').QBIxora -const helper = require('./helper.js') +import {QBIxora} from 'ixora' +import helper from './helper.js' async function visualize_force_graph (req, detected, type) { try { - const graph = new ixora('Social-Analyzer', false) + const graph = new QBIxora('Social-Analyzer', false) const temp_filtered = detected.filter(item => item.status === 'good') if (temp_filtered.length > 0) { if (req.body.group) { @@ -83,6 +83,6 @@ async function visualize_force_graph (req, detected, type) { } } } -module.exports = { +export default{ visualize_force_graph } diff --git a/package-lock.json b/package-lock.json index be431ed..ea59ead 100644 --- a/package-lock.json +++ b/package-lock.json @@ -25,6 +25,7 @@ "langs": "2.0.0", "lodash": ">=4.17.21", "most-common-words-by-language": "3.0.10", + "n": "^8.2.0", "prettier": "2.7.1", "pretty-error": "4.0.0", "sanitize-html": ">=2.7.0", @@ -2987,6 +2988,20 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" }, + "node_modules/n": { + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/n/-/n-8.2.0.tgz", + "integrity": "sha512-qv+jwJoXaV94C+uASaLS/Qe/iprgvQKe+5hqg6YvUH8mqe5ysgjby875pUVLLWieGFywipEO/J/bgekYbC18Jw==", + "os": [ + "!win32" + ], + "bin": { + "n": "bin/n" + }, + "engines": { + "node": "*" + } + }, "node_modules/n-gram": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/n-gram/-/n-gram-2.0.1.tgz", @@ -6504,6 +6519,11 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" }, + "n": { + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/n/-/n-8.2.0.tgz", + "integrity": "sha512-qv+jwJoXaV94C+uASaLS/Qe/iprgvQKe+5hqg6YvUH8mqe5ysgjby875pUVLLWieGFywipEO/J/bgekYbC18Jw==" + }, "n-gram": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/n-gram/-/n-gram-2.0.1.tgz", diff --git a/package.json b/package.json index be41f68..3c20334 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "langs": "2.0.0", "lodash": ">=4.17.21", "most-common-words-by-language": "3.0.10", + "n": "^8.2.0", "prettier": "2.7.1", "pretty-error": "4.0.0", "sanitize-html": ">=2.7.0", @@ -48,5 +49,6 @@ "eslint-plugin-import": "^2.26.0", "eslint-plugin-node": "^11.1.0", "eslint-plugin-promise": "^6.0.0" - } + }, + "type": "module" } diff --git a/public/graph.html b/public/graph.html new file mode 100644 index 0000000..ed8e327 --- /dev/null +++ b/public/graph.html @@ -0,0 +1,42 @@ + + + + +