mirror of
https://github.com/qeeqbox/social-analyzer.git
synced 2026-05-26 05:28:39 +00:00
Giga
842 lines
29 KiB
JavaScript
Executable File
842 lines
29 KiB
JavaScript
Executable File
// -------------------------------------------------------------
|
|
// author Giga
|
|
// project qeeqbox/social-analyzer
|
|
// email gigaqeeq@gmail.com
|
|
// description app.py (CLI)
|
|
// licensee AGPL-3.0
|
|
// -------------------------------------------------------------
|
|
// contributors list qeeqbox/social-analyzer/graphs/contributors
|
|
// -------------------------------------------------------------
|
|
"use strict";
|
|
import yargs from 'yargs'
|
|
import { hideBin } from 'yargs/helpers'
|
|
const yarg_ = yargs(hideBin(process.argv))
|
|
const argv = yarg_.usage('Usage: $0 --username "johndoe" --websites "youtube tiktok"\nUsage: $0 "fast" --username "johndoe"')
|
|
.describe('gui', 'Reserved for a gui')
|
|
.default('gui', false)
|
|
.boolean('gui')
|
|
.describe('cli', 'Reserved for a cli (Not needed)')
|
|
.default('cli', false)
|
|
.boolean('cli')
|
|
.describe('username', 'E.g. johndoe, john_doe or johndoe9999')
|
|
.default('username', '')
|
|
.describe('websites', 'A website or websites separated by space E.g. youtube, tiktok or tumblr')
|
|
.default('websites', 'all')
|
|
.describe('mode', 'Analysis mode E.g.fast -> FindUserProfilesFast, slow -> FindUserProfilesSlow or special -> FindUserProfilesSpecial')
|
|
.default('mode', 'fast')
|
|
.describe('output', 'Show the output in the following format: json -> json output for integration or pretty -> prettify the output')
|
|
.default('output', 'pretty')
|
|
.describe('options', 'Show the following when a profile is found: link, rate, title or text')
|
|
.default('options', '')
|
|
.describe('list', 'List all available websites')
|
|
.default('list', false)
|
|
.boolean('list')
|
|
.describe('docker', 'allow docker')
|
|
.default('docker', false)
|
|
.boolean('docker')
|
|
.describe('method', 'find -> show detected profiles, get -> show all profiles regardless detected or not, all -> combine find & get')
|
|
.default('method', 'all')
|
|
.describe('grid', 'grid option, not for CLI')
|
|
.default('grid', '')
|
|
.describe('extract', 'Extract profiles, urls & patterns if possible')
|
|
.default('extract', false)
|
|
.boolean('extract')
|
|
.describe('metadata', 'Extract metadata if possible (pypi QeeqBox OSINT)')
|
|
.default('metadata', false)
|
|
.boolean('metadata')
|
|
.describe('trim', 'Trim long strings')
|
|
.default('trim', false)
|
|
.boolean('trim')
|
|
.describe('filter', 'filter detected profiles by good, maybe or bad, you can do combine them with comma (good,bad) or use all')
|
|
.default('filter', 'good')
|
|
.describe('profiles', 'filter profiles by detected, unknown or failed, you can do combine them with comma (detected,failed) or use all')
|
|
.default('profiles', 'detected')
|
|
.describe('top', 'select top websites as 10, 50 etc...[--websites is not needed]')
|
|
.default('top', '0')
|
|
.describe('type', 'Select websites by type (Adult, Music etc)')
|
|
.default('type', 'all')
|
|
.describe('countries', 'select websites by country or countries separated by space as: us br ru')
|
|
.default('countries', 'all')
|
|
.help('help')
|
|
.argv
|
|
|
|
if (argv.output !== 'json') {
|
|
console.log('[init] Detections are updated very often, make sure to get the most up-to-date ones')
|
|
}
|
|
|
|
import semver from 'semver'
|
|
|
|
if (semver.satisfies(process.version, '>13 || <13')) {
|
|
if (argv.output !== 'json') {
|
|
console.log('[init] NodeJS Version Check')
|
|
}
|
|
} else {
|
|
if (argv.output !== 'json') {
|
|
console.log('[Error] NodeJS Version Check')
|
|
}
|
|
process.exit(1)
|
|
}
|
|
|
|
import express from 'express'
|
|
import fs from 'fs'
|
|
import tokenizer from 'wink-tokenizer'
|
|
import generatorics from 'generatorics'
|
|
import HttpsProxyAgent from 'https-proxy-agent'
|
|
import PrettyError from 'pretty-error'
|
|
|
|
const pe = new PrettyError()
|
|
import 'express-async-errors'
|
|
//const _tokenizer = tokenizer()
|
|
|
|
if (!fs.existsSync('logs')) {
|
|
fs.mkdirSync('logs')
|
|
}
|
|
|
|
import helper from './modules/helper.js'
|
|
import fastScan from './modules/fast-scan.js'
|
|
import slowScan from './modules/slow-scan.js'
|
|
import specialScan from './modules/special-scan.js'
|
|
import externalApis from './modules/external-apis.js'
|
|
import stringAnalysis from './modules/string-analysis.js'
|
|
import nameAnalysis from './modules/name-analysis.js'
|
|
import visualize from './modules/visualize.js'
|
|
import stats from './modules/stats.js'
|
|
|
|
const app = express()
|
|
app.set('etag', false)
|
|
app.use(express.urlencoded({
|
|
extended: true
|
|
}))
|
|
app.use(express.json())
|
|
app.use(express.static('public'))
|
|
|
|
app.post('/get_logs', async function (req, res, next) {
|
|
let last_line = 'nothinghere'
|
|
if (req.body.uuid !== '') {
|
|
const temp_log_file = helper.get_log_file(req.body.uuid)
|
|
if (fs.existsSync(temp_log_file)) {
|
|
const data = fs.readFileSync(temp_log_file).toString()
|
|
if (typeof data !== 'undefined' && data) {
|
|
last_line = data.split('\n').slice(-2)[0]
|
|
}
|
|
} else {
|
|
last_line = 'nothing_here_error'
|
|
}
|
|
res.send(last_line)
|
|
}
|
|
})
|
|
|
|
app.get('/get_settings', async function (req, res, next) {
|
|
let temp_list = await Promise.all(helper.websites_entries.map(async (site, index) => {
|
|
let temp_url = ''
|
|
if ('status' in site) {
|
|
if (site.status === 'bad') {
|
|
return Promise.resolve()
|
|
}
|
|
}
|
|
if (site.detections.length > 0) {
|
|
temp_url = helper.get_site_from_url(site.url)
|
|
if (temp_url !== 'nothinghere') {
|
|
let temp_selected = 'false'
|
|
if ('selected' in site) {
|
|
if (site.selected === 'true') {
|
|
temp_selected = 'true'
|
|
}
|
|
}
|
|
return Promise.resolve({
|
|
index: index,
|
|
url: temp_url,
|
|
selected: temp_selected,
|
|
global_rank: site.global_rank
|
|
})
|
|
}
|
|
}
|
|
|
|
return Promise.resolve()
|
|
}))
|
|
|
|
temp_list = temp_list.filter(item => item !== undefined)
|
|
temp_list.sort(function (a, b) {
|
|
const keyA = a.url
|
|
const keyB = b.url
|
|
// Compare the 2 dates
|
|
if (keyA < keyB) return -1
|
|
if (keyA > keyB) return 1
|
|
return 0
|
|
})
|
|
res.json({
|
|
proxy: helper.proxy,
|
|
user_agent: helper.header_options.headers['User-Agent'],
|
|
google: [helper.google_api_key.substring(0, 10) + '******', helper.google_api_cs.substring(0, 10) + '******'],
|
|
websites: temp_list
|
|
})
|
|
})
|
|
|
|
app.post('/save_settings', async function (req, res, next) {
|
|
await helper.websites_entries.forEach(function (value, i) {
|
|
helper.websites_entries[i].selected = 'false'
|
|
})
|
|
if ('websites' in req.body) {
|
|
if (req.body.websites.length > 0) {
|
|
await req.body.websites.split(',').forEach(item => {
|
|
helper.websites_entries[Number(item)].selected = 'true'
|
|
})
|
|
}
|
|
}
|
|
if (req.body.google_key !== helper.google_api_key.substring(0, 10) + '******') {
|
|
helper.google_api_key = req.body.google_key
|
|
}
|
|
if (req.body.google_cv !== helper.google_api_cs.substring(0, 10) + '******') {
|
|
helper.google_api_cs = req.body.google_cv
|
|
}
|
|
if (req.body.user_agent !== helper.header_options.headers['User-Agent']) {
|
|
helper.header_options.headers['User-Agent'] = req.body.user_agent
|
|
}
|
|
if (req.body.proxy !== helper.proxy) {
|
|
helper.proxy = req.body.proxy
|
|
}
|
|
|
|
if (helper.proxy !== '') {
|
|
helper.header_options.agent = HttpsProxyAgent(helper.proxy)
|
|
} else {
|
|
if ('agent' in helper.header_options) {
|
|
delete helper.header_options.agent
|
|
}
|
|
}
|
|
|
|
res.json('Done')
|
|
})
|
|
|
|
app.get('/generate', async function (req, res, next) {
|
|
const list_of_combinations = []
|
|
if (req.body.option === 'Generate') {
|
|
if (req.body.words !== undefined && req.body.words.length > 1 && req.body.words.length < 8) {
|
|
for (const perm of generatorics.permutationCombination(req.body.words)) {
|
|
if (perm.join('') !== '') {
|
|
list_of_combinations.push(perm.join(''))
|
|
}
|
|
}
|
|
}
|
|
}
|
|
res.json({
|
|
combinations: list_of_combinations
|
|
})
|
|
})
|
|
|
|
app.post('/cancel', async function (req, res, next) {
|
|
if (req.body.option === 'on' && req.body.uuid !== '') {
|
|
const temp_uuid = req.body.uuid.replace(/[^a-zA-Z0-9\-]+/g, '')
|
|
if (!helper.global_lock.includes(temp_uuid)) {
|
|
helper.log_to_file_queue(req.body.uuid, '[Canceling] task: ' + req.body.uuid)
|
|
helper.global_lock.push(temp_uuid)
|
|
}
|
|
}
|
|
res.json('Done')
|
|
})
|
|
|
|
app.post('/analyze_string', async function (req, res, next) {
|
|
let username = ''
|
|
let temp_uuid = ''
|
|
const info = {
|
|
items: [],
|
|
original: '',
|
|
corrected: '',
|
|
total: 0,
|
|
checking: 'Using ' + req.body.string + ' with no lookups'
|
|
}
|
|
const user_info_normal = {
|
|
data: [],
|
|
type: 'all'
|
|
}
|
|
const user_info_advanced = {
|
|
data: [],
|
|
type: 'all'
|
|
}
|
|
const user_info_special = {
|
|
data: [],
|
|
type: 'all'
|
|
}
|
|
const all_words = {
|
|
prefix: [],
|
|
name: [],
|
|
number: [],
|
|
symbol: [],
|
|
unknown: [],
|
|
maybe: []
|
|
}
|
|
let ages = []
|
|
let names_origins = []
|
|
const words_info = []
|
|
const temp_words = []
|
|
let custom_search = []
|
|
let logs = ''
|
|
let fast = false
|
|
let graph = {
|
|
graph: {
|
|
nodes: [],
|
|
links: []
|
|
}
|
|
}
|
|
|
|
let stats_default = {
|
|
categories: {},
|
|
countries: {}
|
|
}
|
|
|
|
if (req.body.string === 'test_user_2021_2022_') {
|
|
if (fs.existsSync('test.json')) {
|
|
res.json(JSON.parse(fs.readFileSync('test.json', 'utf8')))
|
|
} else {
|
|
res.json('Error')
|
|
}
|
|
} else if (req.body.string === null || req.body.string === '') {
|
|
res.json('Error')
|
|
} else {
|
|
username = req.body.string
|
|
req.body.uuid = req.body.uuid.replace(/[^a-zA-Z0-9\-]+/g, '')
|
|
temp_uuid = req.body.uuid
|
|
|
|
helper.log_to_file_queue(req.body.uuid, '[Setting] Log file name: ' + req.body.uuid)
|
|
|
|
if (req.body.string.includes(',')) {
|
|
req.body.group = true
|
|
helper.log_to_file_queue(req.body.uuid, '[Setting] Multiple usernames: ' + req.body.string)
|
|
} else {
|
|
req.body.group = false
|
|
helper.log_to_file_queue(req.body.uuid, '[Setting] Username: ' + req.body.string)
|
|
}
|
|
|
|
if (req.body.option.includes('FindUserProfilesFast') || req.body.option.includes('GetUserProfilesFast')) {
|
|
fast = true
|
|
helper.log_to_file_queue(req.body.uuid, '[Starting] Checking user profiles normal')
|
|
if (req.body.group) {
|
|
const old_string_1 = req.body.string
|
|
const all_usernames = req.body.string.split(',').map(async item => {
|
|
req.body.string = item
|
|
let temp_arr = await fastScan.find_username_normal(req)
|
|
user_info_normal.data.push(...temp_arr)
|
|
})
|
|
await Promise.all(all_usernames)
|
|
req.body.string = old_string_1
|
|
} else {
|
|
user_info_normal.data = await fastScan.find_username_normal(req)
|
|
}
|
|
|
|
helper.log_to_file_queue(req.body.uuid, '[Done] Checking user profiles normal')
|
|
if (req.body.option.includes('CategoriesStats') || req.body.option.includes('MetadataStats')) {
|
|
helper.log_to_file_queue(req.body.uuid, '[Starting] Generate stats')
|
|
stats_default = await stats.get_stats(req,user_info_normal.data)
|
|
helper.log_to_file_queue(req.body.uuid, '[Done] Generate stats')
|
|
}
|
|
}
|
|
|
|
if (req.body.option.includes('FindUserProfilesSpecial')) {
|
|
if (!fast) {
|
|
helper.log_to_file_queue(req.body.uuid, '[Starting] Checking user profiles special')
|
|
user_info_special.data = await specialScan.find_username_special(req)
|
|
helper.log_to_file_queue(req.body.uuid, '[Done] Checking user profiles special')
|
|
} else {
|
|
helper.log_to_file_queue(req.body.uuid, '[Warning] FindUserProfilesFast with FindUserProfilesSpecial')
|
|
helper.log_to_file_queue(req.body.uuid, '[Skipping] FindUserProfilesSpecial')
|
|
}
|
|
}
|
|
|
|
if (req.body.option.includes('FindUserProfilesSlow') && fast) {
|
|
helper.log_to_file_queue(req.body.uuid, '[Warning] FindUserProfilesFast with FindUserProfilesSlow')
|
|
helper.log_to_file_queue(req.body.uuid, '[Skipping] FindUserProfilesSlow')
|
|
}
|
|
|
|
if (req.body.option.includes('ShowUserProfilesSlow') && fast) {
|
|
helper.log_to_file_queue(req.body.uuid, '[Warning] FindUserProfilesFast with ShowUserProfilesSlow')
|
|
helper.log_to_file_queue(req.body.uuid, '[Skipping] ShowUserProfilesSlow')
|
|
}
|
|
|
|
if ((req.body.option.includes('FindUserProfilesSlow') && !fast) || (req.body.option.includes('ShowUserProfilesSlow') && !fast)) {
|
|
if (!req.body.option.includes('FindUserProfilesSlow')) {
|
|
user_info_advanced.type = 'show'
|
|
} else if (!req.body.option.includes('ShowUserProfilesSlow')) {
|
|
user_info_advanced.type = 'noshow'
|
|
}
|
|
helper.log_to_file_queue(req.body.uuid, '[Starting] Checking user profiles advanced')
|
|
|
|
if (req.body.group) {
|
|
const old_string_2 = req.body.string
|
|
const all_usernames = req.body.string.split(',').map(async item => {
|
|
req.body.string = item
|
|
const temp_arr = await slowScan.find_username_advanced(req)
|
|
user_info_advanced.data.push(...temp_arr)
|
|
})
|
|
await Promise.all(all_usernames)
|
|
req.body.string = old_string_2
|
|
} else {
|
|
user_info_advanced.data = await slowScan.find_username_advanced(req)
|
|
}
|
|
|
|
helper.log_to_file_queue(req.body.uuid, '[Done] Checking user profiles advanced')
|
|
}
|
|
|
|
if (!req.body.group) {
|
|
if (req.body.option.includes('LookUps')) {
|
|
helper.log_to_file_queue(req.body.uuid, '[Starting] Lookup')
|
|
await externalApis.check_engines(req, info)
|
|
helper.log_to_file_queue(req.body.uuid, '[Done] Lookup')
|
|
}
|
|
if (req.body.option.includes('CustomSearch')) {
|
|
helper.log_to_file_queue(req.body.uuid, '[Starting] Custom Search')
|
|
custom_search = await externalApis.custom_search_ouputs(req)
|
|
helper.log_to_file_queue(req.body.uuid, '[Done] Custom Search')
|
|
}
|
|
if (req.body.option.includes("FindOrigins")) {
|
|
helper.log_to_file_queue(req.body.uuid, "[Starting] Finding Origins")
|
|
names_origins = await nameAnalysis.find_origins(req);
|
|
helper.log_to_file_queue(req.body.uuid, "[Done] Finding Origins")
|
|
}
|
|
} else {
|
|
if (req.body.option.includes('FindOrigins')) {
|
|
const old_string_2 = req.body.string
|
|
const all_usernames = req.body.string.split(',').map(async item => {
|
|
helper.log_to_file_queue(req.body.uuid, '[Starting] Finding Origins: ' + item)
|
|
req.body.string = item
|
|
const temp_arr = await nameAnalysis.find_origins(req)
|
|
names_origins.push(...temp_arr)
|
|
helper.log_to_file_queue(req.body.uuid, '[Done] Finding Origins: ' + item)
|
|
})
|
|
await Promise.all(all_usernames)
|
|
req.body.string = old_string_2
|
|
}
|
|
|
|
await stringAnalysis.split_comma(req, all_words)
|
|
}
|
|
|
|
if (req.body.option.includes('SplitWordsByUpperCase')) {
|
|
helper.log_to_file_queue(req.body.uuid, '[Starting] Split by UpperCase')
|
|
await stringAnalysis.split_upper_case(req, all_words)
|
|
helper.log_to_file_queue(req.body.uuid, '[Done] Split by UpperCase')
|
|
}
|
|
if (req.body.option.includes('SplitWordsByAlphabet')) {
|
|
helper.log_to_file_queue(req.body.uuid, '[Starting] Split by Alphabet')
|
|
await stringAnalysis.split_alphabet_case(req, all_words)
|
|
helper.log_to_file_queue(req.body.uuid, '[Done] Split by Alphabet')
|
|
}
|
|
if (req.body.option.includes('FindSymbols')) {
|
|
helper.log_to_file_queue(req.body.uuid, '[Starting] Finding Symbols')
|
|
await stringAnalysis.find_symbols(req, all_words)
|
|
helper.log_to_file_queue(req.body.uuid, '[Done] Finding Symbols')
|
|
}
|
|
if (req.body.option.includes('FindNumbers')) {
|
|
helper.log_to_file_queue(req.body.uuid, '[Starting] Finding Numbers')
|
|
await stringAnalysis.find_numbers(req, all_words)
|
|
helper.log_to_file_queue(req.body.uuid, '[Done] Finding Numbers')
|
|
}
|
|
if (req.body.option.includes('FindAges')) {
|
|
helper.log_to_file_queue(req.body.uuid, '[Starting] Finding Ages')
|
|
ages = await stringAnalysis.guess_age_from_string(req)
|
|
helper.log_to_file_queue(req.body.uuid, '[Done] Finding Ages')
|
|
}
|
|
|
|
req.body.string = req.body.string.toLowerCase()
|
|
|
|
if (req.body.option.includes('ConvertNumbers')) {
|
|
helper.log_to_file_queue(req.body.uuid, '[Starting] Convert Numbers')
|
|
await stringAnalysis.convert_numbers(req, all_words)
|
|
helper.log_to_file_queue(req.body.uuid, '[Done] Convert Numbers')
|
|
}
|
|
|
|
if (req.body.option.includes('LookUps') ||
|
|
req.body.option.includes('WordInfo') ||
|
|
req.body.option.includes('MostCommon') ||
|
|
req.body.option.includes('SplitWordsByUpperCase') ||
|
|
req.body.option.includes('SplitWordsByAlphabet') ||
|
|
req.body.option.includes('FindSymbols') ||
|
|
req.body.option.includes('FindNumbers') ||
|
|
req.body.option.includes('ConvertNumbers')) {
|
|
await stringAnalysis.get_maybe_words(req, all_words)
|
|
await stringAnalysis.analyze_string(req, all_words)
|
|
|
|
Object.keys(all_words).forEach((key) => (all_words[key].length === 0) && delete all_words[key])
|
|
|
|
if (req.body.option.includes('MostCommon')) {
|
|
await stringAnalysis.most_common(all_words, temp_words)
|
|
}
|
|
if (req.body.option.includes('WordInfo')) {
|
|
await externalApis.get_words_info(all_words, words_info)
|
|
}
|
|
} else if (req.body.option.includes('NormalAnalysis@@')) {
|
|
/*
|
|
// var maybe_words = WordsNinja.splitSentence(req.body.string);
|
|
all_words.maybe = maybe_words.filter(function (elem, index, self) {
|
|
return index === self.indexOf(elem)
|
|
})
|
|
list_of_tokens = _tokenizer.tokenize(req.body.string)
|
|
list_of_tokens.forEach(function (item, index) {
|
|
if (item.tag in all_words) {
|
|
all_words[item.tag].push(item.token)
|
|
} else {
|
|
all_words[item.tag] = []
|
|
all_words[item.tag].push(item.token)
|
|
}
|
|
})
|
|
|
|
Object.keys(all_words).forEach((key) => (all_words[key].length === 0) && delete all_words[key])
|
|
*/
|
|
}
|
|
|
|
if (req.body.option.includes('NetworkGraph')) {
|
|
if ('data' in user_info_normal) {
|
|
if (user_info_normal.data.length > 0) {
|
|
if (req.body.option.includes('ExtractMetadata')) {
|
|
helper.log_to_file_queue(req.body.uuid, '[Starting] Network Graph')
|
|
graph = await visualize.visualize_force_graph(req, user_info_normal.data, 'fast')
|
|
helper.log_to_file_queue(req.body.uuid, '[Done] Network Graph')
|
|
} else {
|
|
helper.log_to_file_queue(req.body.uuid, '[Warning] NetworkGraph needs ExtractMetadata')
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
try {
|
|
logs = fs.readFileSync(helper.get_log_file(req.body.uuid), 'utf8')
|
|
} catch {
|
|
|
|
}
|
|
|
|
helper.log_to_file_queue(req.body.uuid, '[Finished] Analyzing: ' + req.body.string + ' Task: ' + req.body.uuid)
|
|
|
|
/*fs.writeFileSync('./test.json', JSON.stringify({
|
|
username: username,
|
|
uuid: temp_uuid,
|
|
info,
|
|
ages: ages,
|
|
table: all_words,
|
|
common: temp_words,
|
|
words_info: words_info,
|
|
user_info_normal: user_info_normal,
|
|
user_info_advanced: user_info_advanced,
|
|
user_info_special: user_info_special,
|
|
names_origins: names_origins,
|
|
custom_search: custom_search,
|
|
graph: graph,
|
|
stats: stats_default,
|
|
logs: logs
|
|
}, null, 2) , 'utf-8');*/
|
|
|
|
res.json({
|
|
username: username,
|
|
uuid: temp_uuid,
|
|
info,
|
|
ages: ages,
|
|
table: all_words,
|
|
common: temp_words,
|
|
words_info: words_info,
|
|
user_info_normal: user_info_normal,
|
|
user_info_advanced: user_info_advanced,
|
|
user_info_special: user_info_special,
|
|
names_origins: names_origins,
|
|
custom_search: custom_search,
|
|
graph: graph,
|
|
stats: stats_default,
|
|
logs: logs
|
|
})
|
|
}
|
|
})
|
|
|
|
app.use((err, req, res, next) => {
|
|
helper.verbose && console.log(' --- Global Error ---')
|
|
helper.verbose && console.log(pe.render(err))
|
|
res.json('Error')
|
|
})
|
|
|
|
app.use((req, res, next) => {
|
|
res.set('Cache-Control', 'no-store')
|
|
next()
|
|
})
|
|
|
|
process.on('uncaughtException', function (err) {
|
|
helper.verbose && console.log(' --- Uncaught Error ---')
|
|
helper.verbose && console.log(pe.render(err))
|
|
})
|
|
|
|
process.on('unhandledRejection', function (err) {
|
|
helper.verbose && console.log(' --- Uncaught Rejection ---')
|
|
helper.verbose && console.log(pe.render(err))
|
|
})
|
|
|
|
function delete_keys (object, temp_keys) {
|
|
temp_keys.forEach((key) => {
|
|
try {
|
|
delete object[key]
|
|
} catch (err) {}
|
|
})
|
|
return object
|
|
}
|
|
|
|
function clean_up_item (object, temp_keys_str) {
|
|
delete object.image
|
|
if (temp_keys_str === '') {} else {
|
|
Object.keys(object).forEach((key) => {
|
|
try {
|
|
if (!temp_keys_str.includes(key)) {
|
|
delete object[key]
|
|
}
|
|
} catch (err) {}
|
|
})
|
|
}
|
|
return object
|
|
}
|
|
|
|
function search_and_change (site, _dict) {
|
|
if (helper.websites_entries.includes(site)) {
|
|
const item = helper.websites_entries.indexOf(site)
|
|
if (item !== -1) {
|
|
helper.websites_entries[item] = Object.assign({}, helper.websites_entries[item], _dict)
|
|
}
|
|
}
|
|
}
|
|
|
|
async function check_user_cli (argv) {
|
|
let ret = []
|
|
const random_string = Math.random().toString(36).substring(2)
|
|
let temp_options = 'GetUserProfilesFast,FindUserProfilesFast'
|
|
if (argv.method !== '') {
|
|
if (argv.method === 'find') {
|
|
temp_options = ',FindUserProfilesFast,'
|
|
} else if (argv.method === 'get') {
|
|
temp_options = ',GetUserProfilesFast,'
|
|
}
|
|
}
|
|
if (argv.extract) {
|
|
temp_options += ',ExtractPatterns,'
|
|
}
|
|
if (argv.metadata) {
|
|
temp_options += ',ExtractMetadata,'
|
|
}
|
|
const req = {
|
|
body: {
|
|
uuid: random_string,
|
|
string: argv.username,
|
|
option: temp_options + argv.output
|
|
}
|
|
}
|
|
|
|
await helper.websites_entries.forEach(async function (value, i) {
|
|
helper.websites_entries[i].selected = 'false'
|
|
})
|
|
|
|
if (argv.websites === 'all') {
|
|
if (argv.countries != 'all') {
|
|
let list_of_countries = argv.countries.toLowerCase().split(' ')
|
|
await helper.websites_entries.forEach(async function (value, i) {
|
|
if (helper.websites_entries[i].country.toLowerCase() !== '' && list_of_countries.includes(helper.websites_entries[i].country.toLowerCase())) {
|
|
helper.websites_entries[i].selected = 'true'
|
|
} else {
|
|
helper.websites_entries[i].selected = 'false'
|
|
}
|
|
})
|
|
} else {
|
|
await helper.websites_entries.forEach(async function (value, i) {
|
|
helper.websites_entries[i].selected = 'true'
|
|
})
|
|
}
|
|
|
|
if (argv.type != 'all') {
|
|
let websites_entries_filtered = helper.websites_entries.filter((item) => item.selected === 'true')
|
|
websites_entries_filtered = websites_entries_filtered.filter((item) => item.type.toLowerCase().includes(argv.type.toLowerCase()))
|
|
|
|
await websites_entries_filtered.forEach(async function (value, i) {
|
|
await search_and_change(websites_entries_filtered[i], {
|
|
selected: 'pendding'
|
|
})
|
|
})
|
|
await helper.websites_entries.forEach(async function (value, i) {
|
|
if (helper.websites_entries[i].selected === 'pendding') {
|
|
helper.websites_entries[i].selected = 'true'
|
|
} else {
|
|
helper.websites_entries[i].selected = 'false'
|
|
}
|
|
})
|
|
}
|
|
|
|
if (argv.top != 0) {
|
|
let websites_entries_filtered = helper.websites_entries.filter((item) => item.selected === 'true')
|
|
websites_entries_filtered = websites_entries_filtered.filter((item) => item.global_rank !== 0)
|
|
websites_entries_filtered.sort(function (a, b) {
|
|
return a.global_rank - b.global_rank
|
|
})
|
|
for (let i = 0; i < argv.top; i++) {
|
|
await search_and_change(websites_entries_filtered[i], {
|
|
selected: 'pendding'
|
|
})
|
|
}
|
|
await helper.websites_entries.forEach(async function (value, i) {
|
|
if (helper.websites_entries[i].selected === 'pendding') {
|
|
helper.websites_entries[i].selected = 'true'
|
|
} else {
|
|
helper.websites_entries[i].selected = 'false'
|
|
}
|
|
})
|
|
}
|
|
} else {
|
|
await helper.websites_entries.forEach(async function (value, i) {
|
|
if (argv.websites.length > 0) {
|
|
await argv.websites.split(' ').forEach(item => {
|
|
if (helper.websites_entries[i].url.toLowerCase().includes(item.toLowerCase())) {
|
|
helper.websites_entries[i].selected = 'true'
|
|
}
|
|
})
|
|
}
|
|
})
|
|
}
|
|
|
|
if (req.body.string.includes(',')) {
|
|
req.body.group = true
|
|
helper.log_to_file_queue(req.body.uuid, '[Setting] Multiple usernames: ' + req.body.string)
|
|
} else {
|
|
req.body.group = false
|
|
helper.log_to_file_queue(req.body.uuid, '[Setting] Username: ' + req.body.string)
|
|
}
|
|
|
|
if (req.body.group) {
|
|
const old_string_1 = req.body.string
|
|
const all_usernames = req.body.string.split(',').map(async item => {
|
|
req.body.string = item
|
|
let temp_arr = await fastScan.find_username_normal(req)
|
|
ret.push(...temp_arr)
|
|
})
|
|
await Promise.all(all_usernames)
|
|
req.body.string = old_string_1
|
|
} else {
|
|
ret = await fastScan.find_username_normal(req)
|
|
}
|
|
|
|
if (typeof ret === 'undefined' || ret === undefined || ret.length === 0) {
|
|
helper.log_to_file_queue(req.body.uuid, 'User does not exist (try FindUserProfilesSlow or FindUserProfilesSpecial)')
|
|
} else {
|
|
const temp_detected = {
|
|
detected: [],
|
|
unknown: [],
|
|
failed: []
|
|
}
|
|
await ret.forEach(item => {
|
|
if (item.method === 'all') {
|
|
if (item.good === 'true') {
|
|
item = delete_keys(item, ['method', 'good'])
|
|
item = clean_up_item(item, argv.options)
|
|
temp_detected.detected.push(item)
|
|
} else {
|
|
item = delete_keys(item, ['found', 'rate', 'status', 'method', 'good', 'text', 'extracted', 'metadata'])
|
|
item = clean_up_item(item, argv.options)
|
|
temp_detected.unknown.push(item)
|
|
}
|
|
} else if (item.method === 'find') {
|
|
if (item.good === 'true') {
|
|
item = delete_keys(item, ['method', 'good'])
|
|
item = clean_up_item(item, argv.options)
|
|
temp_detected.detected.push(item)
|
|
}
|
|
} else if (item.method === 'get') {
|
|
item = delete_keys(item, ['found', 'rate', 'status', 'method', 'good', 'text', 'extracted', 'metadata'])
|
|
item = clean_up_item(item, argv.options)
|
|
temp_detected.unknown.push(item)
|
|
} else if (item.method === 'failed') {
|
|
item = delete_keys(item, ['found', 'rate', 'status', 'method', 'good', 'text', 'language', 'title', 'type', 'extracted', 'metadata'])
|
|
item = clean_up_item(item, argv.options)
|
|
temp_detected.failed.push(item)
|
|
}
|
|
})
|
|
|
|
if (temp_detected.detected.length === 0) {
|
|
delete temp_detected.detected
|
|
} else {
|
|
if (argv.profiles.includes('all') || argv.profiles.includes('detected')) {
|
|
if (argv.filter.includes('all')) {
|
|
|
|
} else {
|
|
temp_detected.detected = temp_detected.detected.filter(item => argv.filter.includes(item.status))
|
|
}
|
|
|
|
if (temp_detected.detected.length === 0) {
|
|
delete temp_detected.detected
|
|
}
|
|
} else {
|
|
delete temp_detected.detected
|
|
}
|
|
}
|
|
|
|
if (temp_detected.unknown.length === 0) {
|
|
delete temp_detected.unknown
|
|
} else {
|
|
if (argv.profiles.includes('all') || argv.profiles.includes('unknown')) {
|
|
|
|
} else {
|
|
delete temp_detected.unknown
|
|
}
|
|
}
|
|
|
|
if (temp_detected.failed.length === 0) {
|
|
delete temp_detected.failed
|
|
} else {
|
|
if (argv.profiles.includes('all') || argv.profiles.includes('failed')) {
|
|
|
|
} else {
|
|
delete temp_detected.failed
|
|
}
|
|
}
|
|
|
|
if (argv.output === 'pretty' || argv.output === '') {
|
|
if ('detected' in temp_detected) {
|
|
helper.log_to_file_queue(req.body.uuid, '[Detected] ' + temp_detected.detected.length + ' Profile[s]')
|
|
helper.log_to_file_queue(req.body.uuid, temp_detected.detected, true, argv)
|
|
}
|
|
if ('unknown' in temp_detected) {
|
|
helper.log_to_file_queue(req.body.uuid, '[Unknown] ' + temp_detected.unknown.length + ' Profile[s]')
|
|
helper.log_to_file_queue(req.body.uuid, temp_detected.unknown, true, argv)
|
|
}
|
|
if ('failed' in temp_detected) {
|
|
helper.log_to_file_queue(req.body.uuid, '[failed] ' + temp_detected.failed.length + ' Profile[s]')
|
|
helper.log_to_file_queue(req.body.uuid, temp_detected.failed, true, argv)
|
|
}
|
|
}
|
|
|
|
if (argv.output === 'json') {
|
|
console.log(JSON.stringify(temp_detected, null, 2))
|
|
}
|
|
}
|
|
};
|
|
|
|
async function list_all_websites () {
|
|
const temp_arr = []
|
|
await helper.websites_entries.forEach(item => {
|
|
temp_arr.push(helper.get_site_from_url(item.url))
|
|
})
|
|
|
|
console.log('[Listing] Available websites\n' + temp_arr.join('\n'))
|
|
}
|
|
|
|
let server_host = 'localhost'
|
|
const server_port = process.env.PORT || 9005
|
|
|
|
if (argv.grid !== '') {
|
|
helper.grid_url = argv.grid
|
|
}
|
|
if (argv.docker) {
|
|
server_host = '0.0.0.0'
|
|
}
|
|
if (argv.gui) {
|
|
app.listen(server_port, server_host, function () {
|
|
// helper.setup_tecert()
|
|
console.log('Server started at http://%s:%s/app.html', server_host, server_port)
|
|
})
|
|
} else {
|
|
if (argv.list) {
|
|
list_all_websites()
|
|
} else if (argv.mode === 'fast') {
|
|
if (argv.cli) {
|
|
console.log('[Warning] --cli is not needed and will be removed later on')
|
|
}
|
|
if (argv.username !== '' && argv.websites !== '') {
|
|
check_user_cli(argv)
|
|
}
|
|
}
|
|
}
|