#!/usr/bin/perl

# Conversion to SQL calculation (not insert into SQL-server):
# 19673 hits, 9565710 bytes, figus 850MHz, 7.87sec, 94% CPU => 7.4 CPU-sec
# 7.4*850000000/19673 = 319727 cpu-cycles/hit
# 1000000000/319727 = 3127 hits/sec @ 1GHz
# => 0.3% CPU-time/hit @ 10 hits/sec @ 1GHz

my $CVS_version =
'$Id$';

my $use_commit = 0;
$use_commit and print "Begin;\n";
while(<>) {
	y/// == (26-1) or do { warn "-- skipping $_\n"; next;}; # 25 ^A'er ellers er den itu
	s/'/''/g;    # Data med ' => ''
	s/\\/\\\\/g;    # Data med \ => \\
	chomp;
	@field = split(//,$_);
	$FormatVersion = shift @field; # 29
	$ServerName = fixServerName(shift @field); # example.com
	$Host = fixHost(shift @field); # www.example.com:80
	$RemoteIP = shift @field; # 12.34.56.78
	$AuthUser = fixAuthUser(shift @field); # -
	$hitDatetime = shift @field; # 2002-01-21 14:54:49 +02:00
	($Method, $Path, $Query, $Protocol) = fixRequest(shift @field); # GET /bl%E5.php?m=bl%E5 HTTP/1.1
	$Filename = shift @field; # /home/hotels/pi.dk/html/index.php3
	$Status = fixStatus(shift @field); # 200
	$BytesSent = fixBytesSent(shift @field); # 458
	$TimeToServe = fixTimeToServe(shift @field); # 1
	$ConnectionStatus = fixConnectionStatus(shift @field); # +
	($Referer,$RefererHost,$RefererPath,$RefererQuery) = fixReferer(shift @field); # http://www.sslug.dk/sponsor.php?q=new
	$UserAgent = fixUserAgent(shift @field); # Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.5) Gecko/20031007
	$AcceptLanguage = fixAcceptLanguage(shift @field); # en-us;q=0.7
	$Accept = fixAccept(shift @field); # text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png
	$AcceptEncoding = fixAcceptEncoding(shift @field); # gzip,deflate
	$AcceptCharset = fixAcceptCharset(shift @field); # ISO-8859-1,utf-8;q=0.7,*;q=0.7
	$Via = fixVia(shift @field); # WTP/1.1 abnws01.sonofon.dk (Nokia WAP Gateway 4.0/ECD10/4.0.64)
	$Cookie = fixCookie(shift @field); # PHPSESSID=705293186a499c4377fd5670f1de5ea9
	$SetCookie = fixSetCookie(shift @field); # Apache=port72.ds1-soeb.adsl.cybercity.dk.39601088521649810; path=/
	$UserTrack = fixUserTrack(shift @field, $Cookie, $SetCookie, $Query); # port72.ds1-soeb.adsl.cybercity.dk.39601088521649810
	$KeepAlive = fixKeepAlive(shift @field); # 300
	$Connection = fixConnection(shift @field); # Keep-Alive
	$TransferEncoding = fixTransferEncoding(shift @field); # chunked
	$ContentType = fixContentType(shift @field); # text/html

	# Check for 
	if($Filename =~ m:/home/hotels/([^/]+): or 
	   # /home/www/public_html/w0.dk/index.php
	   $Filename =~ m:/home/www/public_html/([^/]+): or
	   # /home/vibeke/public_html/vibeke.schou.dk/
	   $Filename =~ m:/home/[^/]+/public_html/([^/]+): or
	   # /home/chlor/public_html
	   $Filename =~ m:/home/([^/]+)/public_html/: or
	   $Filename =~ m:/home/(httpd)/: or
	   $Filename =~ m:.*?/hotel/([^/]+): or
	   $Filename =~ m:.*?/person/([^/]+): or
	   $Path =~ m:^/~([^/]+): or # pi.dk/~tange
	   $ServerName =~ /(.+)/ or
	   $Status == 302 or
	   $Status == 408 or # Time out
	   $Status == 404 or # File not found
	   $Status == 414 or # Too long
	   $Status == 400 or # www.=%&#.dk
	   $Filename =~ /^proxy:/
	   ) { } else { die("Customer not found in $Filename\n$_"); }
	# XXX $CustomerText = $1; # pi.dk
	$CustomerText = $ServerName;
	($SearchEngine,$SearchWords) = fixSearch($Referer);
	print ("select addhit(\n '",
		join("',\n '",
			$FormatVersion,		#  1
			$CustomerText,		#  2
			$ServerName,		#  3
			$Host,			#  4
			$RemoteIP,		#  5
			$AuthUser,		#  6
			$hitDatetime,		#  7
			$Method,		#  8
			$Path,			#  9
			$Query,			# 10
			$Protocol,		# 11
			$Filename,		# 12
			$Status,		# 13
			$BytesSent,		# 14
			$TimeToServe,		# 15
			$ConnectionStatus,	# 16
			$Referer,		# 17
			$RefererHost,		# 18
			$RefererPath,		# 19
			$RefererQuery,		# 20
			$UserAgent,		# 21
			$AcceptLanguage,	# 22
			$Accept,		# 23
			$AcceptEncoding,	# 24
			$AcceptCharset,		# 25
			$Via,			# 26
			$Cookie,		# 27
			$SetCookie,		# 28
			$UserTrack,		# 29
			$ContentType,		# 30
			$ENV{HOSTNAME},		# 31
			join("", $Connection, # 32 needed because PostgreSQL does not support 36 parameters
				$TransferEncoding,
				$KeepAlive,
				$SearchEngine,
				$SearchWords
			)
		), "'\n);\n");
	$use_commit and ++$commit % 100 or do { print "commit;begin;"; };
}
$use_commit and print "Commit;\n";

###########################
#

sub fixServerName {
	return lc shift;
}

sub fixHost {
	# www.example.com[:80] RFC 2616 then strip off ":80"
	my $name = shift;
	$name =~ s/:80$//;
	return $name eq "-" ? "" : $name;
}

sub fixAuthUser {
	my $name = shift;
	return $name eq "-" ? "" : $name;
}

sub fixStatus {
	my $stat = shift;
	return $stat eq "-" ? 0 : $stat;
}

sub fixBytesSent {
	my $bytes = shift;
	return $bytes eq "-" ? 0 : $bytes;
}

sub fixTimeToServe {
	# Integer 0..? What is limit? 3600 sec?
	my $var = shift;
	return $var eq "-" ? "" : $var;
}

sub fixConnectionStatus {
	# Must only one char: + - X
	my $var = shift;
	return $var eq "-" ? "" : $var;
}

sub fixUserAgent {
	my $var = shift;
	return $var eq "-" ? "" : $var;
}

sub fixAcceptLanguage {
	my $var = shift;
	return $var eq "-" ? "" : $var;
}

sub fixAccept {
	my $var = shift;
	return $var eq "-" ? "" : $var;
}

sub fixAcceptEncoding {
	my $var = shift;
	return $var eq "-" ? "" : $var;
}

sub fixAcceptCharset {
	my $var = shift;
	return $var eq "-" ? "" : $var;
}

sub fixVia {
	my $via = shift;
	return $via eq "-" ? "" : $via;
}

sub fixCookie {
	my $cook = shift;
	return $cook eq "-" ? "" : $cook;
}

sub fixSetCookie {
	my $cook = shift;
	return $cook eq "-" ? "" : $cook;
}

sub fixUserTrack {
	my $usertrack = shift;
	my $cookie = shift;
	my $setcookie = shift;
	my $query = shift; # Try find (?|&)PHPSESSID=.. in query
	# Don't try use usertrack at FH XXX
	$usertrack = "";
	if (
		$cookie =~ m:^fe_typo_user=([a-f0-9]+): or	# fe_typo_user=893a1baa3a
		$cookie =~ m:^PHPSESSID=([a-f0-9]+): or		# PHPSESSID=10a94685eb44bf6a33df78b4a8817c1f
		$cookie =~ m:^Apache=([a-fA-F0-9]+): or		# Apache=213.237.41.196.67541091197058968
		$cookie =~ m:^AWSSESSION_ID=([a-z0-9_]+): or	# AWSSESSION_ID=awssession_id1091289549809r7297
		$cookie =~ m:^AWSUSER_ID=([a-z0-9_]+):		# AWSUSER_ID=awsuser_id1091289549809r7297
		#$query =~ m:PHPSESSID=([a-f0-9]+): or		# PHPSESSID=10a94685eb44bf6a33df78b4a8817c1f
    		) {
		$usertrack = $1;
	}
	return $usertrack eq "-" ? "" : $usertrack;
}

sub fixKeepAlive {
	# XXX Must integer in return. if "-" then feed with "" to SQL
	my $var = shift;
	return $var eq "-" ? "0" : $var;
}
sub fixConnection {
	my $var = shift;
	return $var eq "-" ? "" : $var;
}

sub fixTransferEncoding {
	my $var = shift;
	return $var eq "-" ? "" : $var;
}

sub fixContentType {
	my $cont = shift;
	$cont =~ s/^([a-zA-Z0-9\/]+).*$/$1/;
	return $cont eq "-" ? "" : $cont;
}

sub fixReferer {
	my ($referer, $refererhost, $refererpath, $refererquery,$protocol);
	$referer = shift;
	if($referer =~
		# http://www.sslug.dk/sponsor.php?q=linux
		m!([^/]+//)	# http://
		([^/]+)		# www.sslug.dk
		(/[^\?]*)?	# /sponsor.php
		(\?.*)?		# ?q=linux
		!x) {
	    $protocol = $1;
	    $refererhost = lc($2); # www.sslug.dk
	    $refererpath = $3; # /sponsor.php
	    $refererquery = $4; # ?q=linux?h=k
	    $refererpath =~ s://*:/:g; # /// -> /		 
	} else {
	    $protocol = "";
	    $refererhost = ""; # -
	    $refererpath = ""; # 
	    $refererquery = ""; # 
	}
	return ($protocol.$refererhost.$refererpath.$refererquery,
		$refererhost, $refererpath, $refererquery);
}

sub fixRequest {
	my ($request, $reqmethod, $reqpath, $reqquery, $reqprotocol);
	$request = shift;
	# GET /bl%E5.php?m=bl%E5 HTTP/1.1
	$request =~ s/^(\S+) // or syslogerror("request1: $request");
	$reqmethod = $1;
	$request =~ s/ (\S+)$// or syslogerror("request2: $request");
	$reqprotocol = $1;
	$request =~ s/^([^?]+)// or syslogerror("request3: $request");
	$reqpath = $1;
	$reqquery = $request;
	$reqpath =~ s://*:/:g; # /// -> /
	return ($reqmethod, $reqpath, $reqquery, $reqprotocol);
}

# Convert referer to: SearchEngineName, SearchWords
#  Example: "Google", "Linux laptop"
sub fixSearch() {
	my ($searchengine,$searchquery);
	return ("", "");
}

sub syslogerror {
	warn @_; # XXX
}

__END__
select addhit('test','25','w0.dk','w0.dk','213.237.41.196','','2004-07-24 18:24:50 +0200','GET','/~chlor/test/','','HTTP/1.1','/home/chlor/public_html/test/','200','961','0','+','-','','','','Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.5) Gecko/20031007','da,en;q=0.7,es;q=0.3','text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,image/jpeg,image/gif;q=0.2,*/*;q=0.1','gzip,deflate','ISO-8859-1,utf-8;q=0.7,*;q=0.7','-','PHPSESSID=9fab571c9b1acf1f0b7cc03e0f448bb7; Apache=213.237.41.196.adsl.ynoe.worldonline.dk.206011090675553214','-','213.237.41.196.adsl.ynoe.worldonline.dk.223531090686290578','300','keep-alive','chunked','text/html','','');



