#define DOWNLOAD_FILE1				"TOS_Distro.ISO"
#define DOWNLOAD_FILE1_SIZE 		16000000

#define HOURS_MAX 					(24*3)

class LogStruct
{
	LogStruct	*next, *last;
	LogStruct	*ip_num_left, *ip_num_right;
	U32			 ip_num, code;
	I64			 size;
	U8			*file, *link;
	CDate		 datetime;
};

class LinkStruct
{
	LinkStruct	*left, *right;
	U8			*link, *file;
	I64			 count;
};

class BlockedStruct
{
	BlockedStruct	*next, *last;
	U32				 ip_num;
};

U0 LogStructDel(LogStruct *tmplg)
{
	Free(tmplg->file);
	Free(tmplg->link);
	Free(tmplg);
}

U0 ParseSingleLogFile(LogStruct *head, U8 *name, CDate *_dstart, CDate *_dend)
{
	CDoc		*doc = DocRead(name, DOCF_PLAIN_TEXT_TABS | DOCF_DBL_DOLLARS | DOCF_NO_CURSOR);
	CDocEntry	*doc_e = doc->head.next;
	U8			*src, *src2, *mon_list = Define("ST_MONTHS");
	LogStruct	*tmplg;
	CDateStruct	 ds;
	I64 i;

	"%$$Q\n", name;
	while (doc_e != doc)
	{
		if (doc_e->type_u8 == DOCT_TEXT)
		{
			tmplg = CAlloc(sizeof(LogStruct));
			try
			{
				src = doc_e->tag;
				tmplg->ip_num.u8[3] = Str2I64(src, 10, &src);
				if (*src++ != '.') throw;
				tmplg->ip_num.u8[2] = Str2I64(src, 10, &src);
				if (*src++ != '.') throw;
				tmplg->ip_num.u8[1] = Str2I64(src, 10, &src);
				if (*src++ != '.') throw;
				tmplg->ip_num.u8[0] = Str2I64(src, 10, &src);

				do if (!*src) throw;
				while (*src++ != '[');
				MemSet(&ds, 0, sizeof(CDateStruct));
				ds.day_of_mon = Str2I64(src, 10, &src);
				if (*src++ != '/') throw;
				src2 = src;
				do if (!*src2) throw;
				while (*src2++ != '/');
				* --src2 = 0;
				ds.mon = 1 + ListMatch(src, mon_list, LMF_IGNORE_CASE);
				src = ++src2;
				ds.year = Str2I64(src, 10, &src);
				if (*src++ != ':') throw;
				ds.hour = Str2I64(src, 10, &src);
				if (*src++ != ':') throw;
				ds.min = Str2I64(src, 10, &src);
				if (*src++ != ':') throw;
				ds.sec = Str2I64(src, 10, &src);
				tmplg->datetime = Struct2Date(&ds);
				if (*src++ != CH_SPACE) throw;
				i = Str2I64(src, 10, &src);
				tmplg->datetime -= (i / 100 + i % 100 / 60.0) * CDATE_FREQ * 60 * 60;
				if (!(Str2Date("1/1/2017") <= tmplg->datetime < Str2Date("1/1/2050")))
					throw;
				if (tmplg->datetime < *_dstart)	*_dstart = tmplg->datetime;
				if (tmplg->datetime > *_dend) 	*_dend	 = tmplg->datetime;

				do if (!*src) throw;
				while (*src++ != ']');

				if (*src++ != CH_SPACE) throw;
				if (*src++ != '\"') throw;
				if (!StrNCompare(src, "GET ", 4))
				{
					src2 = src += 4;
					do if (!*src2) throw;
					while (*src2++ != CH_SPACE);
					* --src2 = 0;
					tmplg->file = StrNew(src);
					src = ++src2;

					do if (!*src) throw;
					while (*src++ != '\"');

					tmplg->code = Str2I64(src, 10, &src);
					if (*src++ != CH_SPACE) throw;
					tmplg->size = Str2I64(src, 10, &src);
					if (*src++ != CH_SPACE) throw;

					if (*src++ != '\"') throw;
					src2 = src;
					do if (!*src2) throw;
					while (*src2++ != '\"');
					* --src2 = 0;
					tmplg->link = StrNew(src);
					src = ++src2;

					QueueInsert(tmplg, head->last);
				}
				else if (!StrNCompare(src, "HEAD ", 5))
				{
					LogStructDel(tmplg);
				}
				else
					throw;
			}
			catch
			{
				Fs->catch_except = TRUE;
				"%$$Q\n", doc_e->tag;
				LogStructDel(tmplg);
			}
		}
		doc_e = doc_e->next;
	}
	DocDel(doc);
}

LogStruct *ParseLogFiles(U8 *files_find_mask, CDate *_dstart, CDate *_dend)
{
	LogStruct *head = CAlloc(sizeof(LogStruct));
	CDirEntry *tmpde = FilesFind(files_find_mask), *tmpde1 = tmpde;

	QueueInit(head);
	while (tmpde)
	{
		ParseSingleLogFile(head, tmpde->full_name, _dstart, _dend);
		tmpde = tmpde->next;
	}
	DirTreeDel(tmpde1);

	return head;
}

U0 LogListDel(LogStruct *head)
{
	LogStruct *tmplg = head->next, *tmplg1;

	while (tmplg != head)
	{
		tmplg1 = tmplg->next;
		LogStructDel(tmplg);
		tmplg = tmplg1;
	}
}

U0 BlockedStructAdd(BlockedStruct *head, U32 ip_num)
{
	BlockedStruct *tmpb = CAlloc(sizeof(BlockedStruct));

	tmpb->ip_num = ip_num;
	QueueInsert(tmpb, head->last);
}

Bool IsBlocked(BlockedStruct *head, U32 ip_num)
{
	BlockedStruct *tmpb = head->next;

	while (tmpb != head)
	{
		if (tmpb->ip_num == ip_num)
			return TRUE;
		tmpb = tmpb->next;
	}

	return FALSE;
}

U0 BlockIPNuip(LogStruct *head)
{
	BlockedStruct blocked_head;
	LogStruct	 *tmplg = head->next, *tmplg1;

	QueueInit(&blocked_head);

	BlockedStructAdd(&blocked_head, 68 << 24 + 227 << 16 + 61 << 8 + 6);

	//pass 1: collect robot list
	while (tmplg != head)
	{
		if (StrIMatch("ROBOT", tmplg->file) && !IsBlocked(&blocked_head, tmplg->ip_num))
			BlockedStructAdd(&blocked_head, tmplg->ip_num);
		tmplg = tmplg->next;
	}

	//pass 2: removed blocked ip_nuip
	tmplg = head->next;
	while (tmplg != head)
	{
		tmplg1 = tmplg->next;
		if (IsBlocked(&blocked_head, tmplg->ip_num))
		{
			QueueRemove(tmplg);
			LogStructDel(tmplg);
		}
		tmplg = tmplg1;
	}

	QueueDel(&blocked_head);
}

Bool IsDownLoad(LogStruct *tmplg)
{
	if (StrMatch(DOWNLOAD_FILE1, tmplg->file)&&tmplg->size >= DOWNLOAD_FILE1_SIZE)
		return TRUE;
	else
		return FALSE;
}

Bool IsIndex(LogStruct *tmplg)
{
	if (!StrCompare(tmplg->file, "/index.html") || !StrCompare(tmplg->file, "/"))
		return TRUE;
	else
		return FALSE;
}

Bool IsKeeper(LogStruct *tmplg, CDate dstart, CDate dend)
{
	if (dstart <= tmplg->datetime <= dend									&&
		!StrOcc(tmplg->file, '?')											&&
		StrLen(tmplg->file) > 2												&&
		'A' <= tmplg->file[1] <= 'Z'										&&
		tmplg->size															&&
		tmplg->file[StrLen(tmplg->file) - 1] != '/'							&&
		(StrLen(tmplg->file) < 3 || MemCompare(&tmplg->file[1], "Wb", 2))	&&
		(StrLen(tmplg->file) < 7 || MemCompare(&tmplg->file[1], "Family", 6)))
	{
		return TRUE;

}  else
		return FALSE;
}

Bool IPNumTreeAdd(LogStruct **_head, LogStruct *tmplg)
{
	LogStruct *head;

	if (UnusedStack < 0x200)
	{
		PrintErr("Stack Overflow");
		throw;
	}
	if (head = *_head)
	{
		if (tmplg->ip_num == head->ip_num)
			return TRUE;
		else if (tmplg->ip_num < head->ip_num)
			return IPNumTreeAdd(&head->ip_num_left, tmplg);
		else
			return IPNumTreeAdd(&head->ip_num_right, tmplg);
	}
	else
	{
		tmplg->ip_num_left = NULL;
		tmplg->ip_num_right = NULL;
		*_head = tmplg;

		return FALSE;
	}
}

U0 LinkTreeAdd(LinkStruct **_root, LogStruct *tmplg)
{
	I64			i;
	LinkStruct *root, *tmplk;

	if (UnusedStack < 0x200)
	{
		PrintErr("Stack Overflow");
		throw;
	}
	if (root = *_root)
	{
		if (!(i = StrCompare(tmplg->link, root->link)))
			root->count++;
		else if (i < 0)
			LinkTreeAdd(&root->left, tmplg);
		else
			LinkTreeAdd(&root->right, tmplg);
	}
	else
	{
		tmplk = CAlloc(sizeof(LinkStruct));
		tmplk->link  = tmplg->link;
		tmplk->count = 1;
		*_root = tmplk;
	}
}

U0 FileTreeAdd(LinkStruct **_root, LogStruct *tmplg)
{
	I64			i;
	LinkStruct *root, *tmplk;
	if (UnusedStack < 0x200)
	{
		PrintErr("Stack Overflow");
		throw;
	}
	if (root = *_root)
	{
		if (!(i = StrCompare(tmplg->file, root->file)))
			root->count++;
		else if (i < 0)
			FileTreeAdd(&root->left, tmplg);
		else
			FileTreeAdd(&root->right, tmplg);
	}
	else
	{
		tmplk = CAlloc(sizeof(LinkStruct));
		tmplk->file  = tmplg->file;
		tmplk->count = 1;
		*_root = tmplk;
	}
}

U0 LinkTreeDel(LinkStruct *root)
{
	if (root)
	{
		LinkTreeDel(root->left);
		LinkTreeDel(root->right);
		Free(root);
	}
}

U0 LinkTreeTraverse(LinkStruct *root)
{
	if (root)
	{
		LinkTreeTraverse(root->left);
		"%3d:%$$Q\n", root->count, root->link;
		LinkTreeTraverse(root->right);
	}
}

U0 FileTreeDel(LinkStruct *root)
{
	if (root)
	{
		FileTreeDel(root->left);
		FileTreeDel(root->right);
		Free(root);
	}
}

U0 FileTreeTraverse(LinkStruct *root)
{
	if (root)
	{
		FileTreeTraverse(root->left);
		"%3d:%$$Q\n", root->count, root->file;
		FileTreeTraverse(root->right);
	}
}

U0 DownLoadRep(LogStruct *head, CDate dstart, CDate dend)
{
	I64			i, j, count, dups, 
				hours_start, hours_end, *hour_counts, *dup_counts, 
				days_start, days_end, *day_counts, *day_dup_counts;
	LogStruct  *tmplg = head->next, *dup_head = NULL;
	LinkStruct *link_root = NULL;
	CDateStruct	ds;

	i = dstart	* 24;
	hours_start = i.u32[1];
	i = dend	* 24;
	hours_end	= i.u32[1];

	days_start	= (dstart + local_time_offset) >> 32;
	days_end	= (dend   + local_time_offset) >> 32;

	hour_counts		= CAlloc((hours_end - hours_start + 1) * sizeof(I64));
	dup_counts		= CAlloc((hours_end - hours_start + 1) * sizeof(I64));
	day_counts		= CAlloc((days_end  - days_start  + 1) * sizeof(I64));
	day_dup_counts	= CAlloc((days_end  - days_start  + 1) * sizeof(I64));
	dups = count = 0;
	while (tmplg != head)
	{
		if (IsKeeper(tmplg, dstart, dend) && IsDownLoad(tmplg))
		{
			i = tmplg->datetime * 24;
			hour_counts[i.u32[1] - hours_start]++;
			day_counts[(tmplg->datetime + local_time_offset) >> 32 - days_start]++;
			count++;
			if (IPNumTreeAdd(&dup_head, tmplg))
			{
				day_dup_counts[(tmplg->datetime + local_time_offset) >> 32 - days_start]++;
				dup_counts[i.u32[1] - hours_start]++;
				dups++;
			}
			LinkTreeAdd(&link_root, tmplg);
		}
		tmplg = tmplg->next;
	}

	"\n\nDownloads of /TOS_Distro.ISO\n";
	for (i = dstart; i <= dend; i += 1 << 32)
		"%D  Dups:%5d  Total:%5d  Uniques:%5d\n", i, 
		day_dup_counts[(i + local_time_offset) >> 32 - days_start], 
		day_counts[(i + local_time_offset) >> 32 - days_start], 
		day_counts[(i + local_time_offset) >> 32 - days_start] - day_dup_counts[(i + local_time_offset) >> 32 - days_start];

	"\n\nDownloads of /TOS_Distro.ISO\n"
	"'-' is a dup.  '+' is not a dup.\n";
	if (hours_end - hours_start >= HOURS_MAX)
		i = hours_end - HOURS_MAX + 1;
	else
		i = hours_start;
	for (; i <= hours_end; i++)
	{
		Date2Struct(&ds, i << 32 / 24 + local_time_offset);
		"%D %02d: ", i << 32 / 24, ds.hour;
		for (j = 0; j < dup_counts[i - hours_start]; j++)
			'-';
		for (; j < hour_counts[i - hours_start]; j++)
			'+';
		'\n';
	}
	"Total:%d  Dups:%d  Uniques:%d\n", count, dups, count - dups;

	"\n\nDownloads of /TOS_Distro.ISO\n";
	LinkTreeTraverse(link_root);
	'\n';

	LinkTreeDel(link_root);
	Free(hour_counts);
	Free(dup_counts);
	Free(day_counts);
	Free(day_dup_counts);
}

U0 FileRep(LogStruct *head, CDate dstart, CDate dend)
{
	LogStruct  *tmplg = head->next;
	LinkStruct *file_root = NULL;

	while (tmplg != head)
	{
		if (IsKeeper(tmplg, dstart, dend))
			FileTreeAdd(&file_root, tmplg);
		tmplg = tmplg->next;
	}
	"\n\nFile Hits\n";
	FileTreeTraverse(file_root);
	'\n';
	FileTreeDel(file_root);
}

U0 IndexRep(LogStruct *head, CDate dstart, CDate dend)
{
	I64			i, j, count, dups, 
				hours_start, hours_end, *hour_counts, *dup_counts, 
				days_start, days_end, *day_counts, *day_dup_counts;
	LogStruct  *tmplg = head->next, *dup_head = NULL;
	LinkStruct *link_root = NULL;
	CDateStruct ds;

	i = dstart	* 24;
	hours_start	= i.u32[1];
	i = dend	* 24;
	hours_end	= i.u32[1];

	days_start	= (dstart + local_time_offset) >> 32;
	days_end	= (dend   + local_time_offset) >> 32;

	hour_counts		= CAlloc((hours_end - hours_start + 1) * sizeof(I64));
	dup_counts		= CAlloc((hours_end - hours_start + 1) * sizeof(I64));
	day_counts		= CAlloc((days_end	- days_start  + 1) * sizeof(I64));
	day_dup_counts	= CAlloc((days_end	- days_start  + 1) * sizeof(I64));
	dups = count = 0;
	while (tmplg != head)
	{
		if (IsKeeper(tmplg, dstart, dend) && IsIndex(tmplg))
		{
			i = tmplg->datetime * 24;
			hour_counts[i.u32[1] - hours_start]++;
			day_counts[(tmplg->datetime + local_time_offset) >> 32 - days_start]++;
			count++;
			if (IPNumTreeAdd(&dup_head, tmplg))
			{
				day_dup_counts[(tmplg->datetime + local_time_offset) >> 32 - days_start]++;
				dup_counts[i.u32[1] - hours_start]++;
				dups++;
			}
			LinkTreeAdd(&link_root, tmplg);
		}
		tmplg = tmplg->next;
	}

	"\n\nHits on /index.html\n"
	"'-' is a dup.  '+' is not a dup.\n";
	for (i = dstart; i <= dend; i += 1 << 32)
		"%D  Dups:%5d  Total:%5d  Uniques:%5d\n", i, 
		day_dup_counts[(i + local_time_offset) >> 32 - days_start], 
		day_counts[(i + local_time_offset) >> 32 - days_start], 
		day_counts[(i + local_time_offset) >> 32 - days_start] - day_dup_counts[(i + local_time_offset) >> 32 - days_start];

	"\n\nHits on /index.html\n";
	if (hours_end - hours_start >= HOURS_MAX)
		i = hours_end - HOURS_MAX + 1;
	else
		i = hours_start;
	for (; i <= hours_end; i++)
	{
		Date2Struct(&ds, i << 32 / 24 + local_time_offset);
		"%D %02d: ", i << 32 / 24, ds.hour;
		for (j = 0; j < dup_counts[i - hours_start];j++)
			'-';
		for (; j < hour_counts[i - hours_start]; j++)
			'+';
		'\n';
	}
	"Total:%d  Dups:%d  Uniques:%d\n", count, dups, count - dups;

	"\n\nHits on /index.html\n";
	LinkTreeTraverse(link_root);
	'\n';

	LinkTreeDel(link_root);
	Free(hour_counts);
	Free(dup_counts);
	Free(day_counts);
	Free(day_dup_counts);
}

U0 WebLogRep(U8 *mask, U8 *output_filename)
{
	LogStruct  *head;
	CDate		dstart = I64_MAX, dend = I64_MIN;

	DocMax;

	head = ParseLogFiles(mask, &dstart, &dend);
	if (dstart > dend)
		PrintErr("No Data.\n");
	else
	{
		dstart	= DateGet("Start(%D):", dstart);
		dend	= DateGet("End  (%D):", dend);
		BlockIPNuip(head);

		DocClear;
		"$$WW,0$$";
		IndexRep(head, dstart, dend);
		FileRep(head, dstart, dend);
		DownLoadRep(head, dstart, dend);

		StrCopy(DocPut->filename.name, output_filename);
		DocWrite(DocPut, TRUE);

		"$$WW,1$$";
	}
	LogListDel(head);
}

#if __CMD_LINE__
Cd(__DIR__);;
WebLogRep("*.log*", "~/DemoWebLog.DD");
#endif
