#!/usr/bin/perl

# Apache-logformat:
# LogFormat "%f^A\n%h^A\n%l^A\n%u^A\n%{%G-%m-%d %T}t^A\n%{Host}i^A\n%U^A\n
# 	     %>s^A\n%b^A\n%{Referer}i^A\n%{User-Agent}i^A\n%T^A\n
# 	     %{Content-Type}o^B\n" perlsplitable
# 
#
# Input:
# /home/hotels/domain123.gov/html/index.htm^A
# 195.47.141.241^A
# -^A
# -^A
# 2000-04-26 20:21:31^A
# www.domain123.gov^A
# /^A
# 200^A
# 544^A
# http://soeg.jubii.dk/res.asp?startnummer=1&default=new&kategori=&soegeord=techno+t%F8j^A
# Mozilla/4.0 (compatible; MSIE 5.0; Windows 95; DigExt)^A
# 1^A
# text/html^B   


# Output:
# select AddHit(
# 'domain123.gov',		# Customername
# '195.47.141.241',		# Client IP    
# '-',				# AuthName     
# '-',				# User-auth    
# '2000-04-26 20:21:31',	# Date-time    
# 'www.domain123.gov',		# Host	       
# '/',				# URL	       
# '200',			# Status       
# '544',			# Size	       
# 'soeg.jubii.dk',		# Referer-host 
# '/res.asp?startnummer=1&default=new&kategori=&soegeord=techno+t%F8j',
#				# Referer-url
# 'Mozilla/4.0 (compatible; MSIE 5.0; Windows 95; DigExt)',
#				# Browser
# '1',				# XferTime     
# 'text/html'			# MimeType
# );

print "begin;";

$/="\002\n\n"; 
while(<>) {
	y/\001/\001/ == (13-1) or next; # 13 fields per line - else skip
	chop;chop;chop; # Remove ^B\n\n
	s/'/''/g; # '-quoting for SQL
	s/\\/\\\\/g; # \-quoting for SQL # bør vi bare s:\\:/: ?
	($customer, $client, $user, $auth, $datetime, $host, $url,
	 $status, $size, $referer, $browser, $xfertime, $mime) 
	    = split("\001\n",$_); # Split on ^A\n

	if($customer=~m:/hotels/([^/]+):) { $customer=$1; }
	elsif($customer=~m:/home/httpd/:) { $customer="-"; }
	elsif($url=~m:/~([^/]+):) { $customer=$1; }
	else { s/\n//g; warn("Warn: Cannot find customer. Skipping:\n$_\n"); next; }

	$host=~y[A-Z][a-z];

	$size=~s/^-$/0/;

	$referer=~m!^(?:[a-z]+://		# http://
		      ([^/]+)			# www.nedergaard.com
		      (\S+)			# /abc.html
		      |
		      ()(.*)			# [unknown origin]
		      )$!x
			  || die ("Referer \n$_\n");
	$referer_host=$1;$referer_host=~y[A-Z][a-z];
	$referer_url=$2;


	# Find TimeSpent if this this a page (I.e. not a picture) and same host
	$find_timespent=($mime=~/text/ and $referer_host eq $host);

	# Sanity check
	$size=~/^\d+$/ || die;
	$status=~/^\d+$/ || die;

	print "select AddHit('",
	join("', '",$customer, $client, $user, $auth, $datetime,
	     $host, $url, $status, $size, 
	     $referer_host, $referer_url, $browser,
	     $xfertime, $mime
	     ),
		 ,"');\n";
	if (not ($line++ % 1000)) { 
	    print "commit;begin;";
	    $time=time;
	    warn($line," ",$time-$lasttime,"\n");
	    $lasttime=$time;
	}
}

print "commit;";

#
# if $find_timespent
# referer_hitID = hitid of hit where
#   $customer=current.$customer
#   $client = current.$client
#   $url = current.$referer_url # Virker dette i praksis?
#   $browser = current.$browser
# timespent = $datetime of current - $datetime of hit(referer_hitID)
# Update hit($referer_hitID, timespent)

__END__

select AddHit('powerplay.dk', '130.226.123.128', '-', '-', '2000-08-03 08:36:23', 'www.powerplay.dk', '/psx-uz.htm', '200', '15713', 'www.powerplay.dk', '/psx-top.htm', 'Mozilla/4.0 (compatible; MSIE 4.01; Windows 95)', '0', '1');

select AddHit('powerplay.dk', '130.226.123.128', '-', '-', '2000-08-04 08:06:03', 'www.powerplay.dk', '/psx-uz.htm', '200', '15713', 'www.powerplay.dk', '/psx-top.htm', 'Mozilla/4.0 (compatible; MSIE 4.01; Windows 95)', '0', '1');


Find de fejlende record i lille.bz2
Prøv:
Find linje med 2000-08-02 09:21:08
herefter find alle linjer med 404

####Fejl 1:
Warn: Cannot find customer. Skipping:
-
203.141.164.23
-
-
2000-08-02 09:21:08
-
-
408
-
-
-
0
-
ERROR:  parser: parse error at or near "404"
NOTICE:  current transaction is aborted, queries ignored until end of transaction block
NOTICE:  current transaction is aborted, queries ignored until end of transaction block
NOTICE:  current transaction is aborted, queries ignored until end of transaction block
NOTICE:  current transaction is aborted, queries ignored until end of transaction block
Warn: Cannot find customer. Skipping:
-
195.54.87.162
-
-
2000-08-02 10:19:35
-
-
408
-
-
-
0
-


####Fejl 2:
Warn: Cannot find customer. Skipping:
-
195.204.240.216
-
-
2000-08-02 16:37:01
-
-
408
-
-
-
0
-
\l: extra argument 'ogovolf.gif',' ignored
ERROR:  parser: parse error at or near "404"
NOTICE:  current transaction is aborted, queries ignored until end of transaction block
NOTICE:  current transaction is aborted, queries ignored until end of transaction block
NOTICE:  current transaction is aborted, queries ignored until end of transaction block
Warn: Cannot find customer. Skipping:
html/bykort.htm
195.215.69.10
-
-
2000-08-02 17:22:36
www.hjortshoj.dk
html/bykort.htm
400
347
-
MindCrawler v4.0,  support@mindpass.com
0
text/html


###
Warn: Cannot find customer. Skipping:
-
207.126.112.254
-
-
2000-08-03 06:20:19
easy_to_use_software_component_07192000.asp
/
400
352
-
Marvin v0.3
0
text/html
invalid command \',
ERROR:  parser: parse error at or near "404"
NOTICE:  current transaction is aborted, queries ignored until end of transaction block
Warn: Cannot find customer. Skipping:
-
63.197.178.26
-
-
2000-08-03 08:59:26
-
/cgi-bin/submit
400
351
-
-
0
text/html


####

Warn: Cannot find customer. Skipping:
-
216.72.201.168
-
-
2000-08-03 10:22:08
-
-
408
-
-
-
0
-
invalid command \kanin-menu2a.GIF',
ERROR:  parser: parse error at or near "404"
NOTICE:  current transaction is aborted, queries ignored until end of transaction block
NOTICE:  current transaction is aborted, queries ignored until end of transaction block
Warn: Cannot find customer. Skipping:
-
193.230.253.23
-
-
2000-08-03 11:01:18
-
-
408
-
-
-
0
-

####
Warn: Cannot find customer. Skipping:
-
207.126.112.254
-
-
2000-08-03 11:36:35
continentalremediation.com\stm-strp.htm
/
400
348
-
Marvin v0.3
0
text/html
invalid command \news01.gif',
ERROR:  parser: parse error at or near "404"
NOTICE:  current transaction is aborted, queries ignored until end of transaction block
NOTICE:  current transaction is aborted, querie
Warn: Cannot find customer. Skipping:
-
207.126.112.254
-
-
2000-08-03 11:56:00
www.gadgets-galore.netInfrared_Cable
/infrared_cable.html
400
345
-
Marvin v0.3
0
text/html

####
Warn: Cannot find customer. Skipping:
formaal.htm
195.215.69.10
-
-
2000-08-03 14:21:42
www.llg.dk
formaal.htm
400
337
-
MindCrawler v4.0,  support@mindpass.com
0
text/html
invalid command \news01.gif',
ERROR:  parser: parse error at or near "404"
NOTICE:  current transaction is aborted, queries ignored until end of transaction block
Warn: Cannot find customer. Skipping:
-
129.240.17.211
-
-
2000-08-03 15:17:22
-
-
408
-
-
-
0
-

####

Warn: Cannot find customer. Skipping:
_vti_bin/shtml.exe/schmidt/top.htm/map
195.215.69.10
-
-
2000-08-03 15:57:56
www.schmidtkontormaskiner.dk
_vti_bin/shtml.exe/schmidt/top.htm/map
400
382
-
MindCrawler v4.0,  support@mindpass.com
0
text/html
ERROR:  parser: parse error at or near "404"
NOTICE:  current transaction is aborted, queries ignored until end of transaction block
NOTICE:  curre
Warn: Cannot find customer. Skipping:
bestyr.htm
195.215.69.10
-
-
2000-08-03 16:25:02
www.llg.dk
bestyr.htm
400
336
-
MindCrawler v4.0,  support@mindpass.com
0
text/html

####

Warn: Cannot find customer. Skipping:
-
212.151.81.171
-
-
2000-08-03 17:55:25
www.kure.dk/mle
/mle/
400
336
http://www.dukker.dk/adresser/link.asp?land=danm
Mozilla/4.0 (compatible; MSIE 4.01; Windows 98)
0
text/html
invalid command \bogen.htm',
ERROR:  parser: parse error at or near "404"
NOTICE:  current transaction is aborted, queries                                n block
NOTICE:  current transaction is aborted, queries                                n block
Warn: Cannot find customer. Skipping:
-
207.126.112.254
-
-
2000-08-03 18:29:32
www.gadgets-galore.netOffer_Page
/offer_page.html
400
341
-
Marvin v0.3
0
text/html


####

Warn: Cannot find customer. Skipping:
-
207.126.112.254
-
-
2000-08-03 18:29:32
www.gadgets-galore.netOffer_Page
/offer_page.html
400
341
-
Marvin v0.3
0
text/html

NOTICE:  current transaction is aborted, queries                                n block
Warn: Cannot find customer. Skipping:
-
207.126.112.254
-
-
2000-08-03 19:07:02
www.gadgets-galore.netPrice_List
/price_list.html
400
341
-
Marvin v0.3
0
text/html

