#!/usr/bin/perl -w
#
# script6: A tool to make complex IPv6 tasks easy
#


$SI6_TOOLKIT="SI6 Networks IPv6 Toolkit v4.0";
$SCRIPT6="script6: A tool to make complex IPv6 tasks easy";
$USAGE="\
Usage: script6 COMMAND [PARAMETERS]
   COMMANDS:
   get-a: Get A records for domain name
   get-aaaa: Get AAAA records for domain name
   get-addresses: Gets A and AAAA records for a domain name
   get-mx: Get X records for domain name
   get-ns: Get NS records fo domain name
   get-asn6: Get Autonomos System Number for IPv6 address
   get-as6: Get Autonomous System information for IPv6 address
   get-ipv6-addrs: Get IPv6 addresses from STDIN
   get-ipv6-prefix: Get IPv6 prefixes from STDIN
   get-alexa-domains: Obtain list of domain names from Alexa's file
   get-majestic-domains: Obtain list of domain names from Majestic's file
   get-bing: Search for domains with the Bing search engine
   get-bing-batch: Search for domains with the Bing search engine
   get-bing-dict: Search for domains with Bing using a dictionary
   get-bing-dict-batch: Search for domains with Bing (STDIN)
   get-bing4: Search for domains with the Bing search engine (IPv4)
   get-bing4-batch: Search for domains with the Bing search engine (IPv4)
   get-bing4-dict: Search for domains with Bing using a dictionary (IPv4)
   get-bing4-dict-batch: Search for domains with Bing (STDIN) (IPv4)
   get-bing6: Search for domains with the Bing search engine (IPv6)
   get-bing6-batch: Search for domains with the Bing search engine (IPv6)
   get-bing6-dict: Search for domains with Bing using a dictionary (IPv6)
   get-bing6-dict-batch: Search for domains with Bing (STDIN) (IPv6)
   get-bing6-addr-dict-quick: Search domains with Bing with custom address
   get-bing6-addr-dict-batch: Search domains with Bing with cust. addr (batch)
   get-bing6-addr-dict-quick-batch: Search dom, w/ Bing w/ cust. addr (batch)
   get-bing-by-addr: Search domains for IPv4 address
   get-bing-by-addr-quick: Search domains for IPv4 address (faster give-up)
   get-bing-by-addr-batch: Search domains for IPv4 address (batch)
   get-crt: Search for domains with crt.sh
   get-crt-batch: Search for domains with crt.sh (STDIN)
   get-crt-addr: Search for domains with crt.sh, from specific address
   get-crt-addr-batch: Search for domains with crt.sh (STDIN), from specific addr
   get-crt-batch-files: Search domains with crt.sh (STDIN)
   get-crt4-batch: Search for domains with crt.sh (STDIN), with IPv4
   get-crt6-batch: Search for domains with crt.sh (STDIN), with IPv6
   get-crt4-batch-files: Search domains with crt.sh (STDIN), with IPv4
   get-crt6-batch-files: Search domains with crt.sh (STDIN), with IPv6
   get-teoma: Search for domains with the Teoma search engine
   get-teoma-batch: Search domains with the Teoma search engine (STDIN)
   get-suffix-for-tld: Obtain known suffixes for TLD
   get-iso2-for-rir: Obtain ISO2 country codes for members of specifi RIR
   get-iso2-for-country: Obtain two-letter ISO code for a country
   get-iso3-for-country: Get ISO3 for country
   get-tld-for-country: Obtain TLD for a given country
   get-tld-for-continent: Obtain TLDs for a given continent
   get-tld-for-rir: Obtain TLDs for a specific RIR
   get-rir-database: Obtain RIR database
   get-country-for-continent: Obtain list of countries for a given continent
   get-dns-zones: Obtain DNS zones in separate files
   get-dns6-rev-batch: Batch enumaration based on DNS reverse mappings
   get-root-zone: Obtain list of all TLDs from IANA
   get-root-zone-raw: Obtain list of all TLDs from IANA (raw format)
   get-ukgov-zone: Obtains the list of domains for the uk.gov.uk zone
   get-trace6: Perform traceroute6 (with EHs) to a given destination
   get-trace6-stats: Get statistics for trace results read from STDIN
   create-dict: Extracts words from STDIN to create a dictionary
   get-unique-domains: Creates a list of unique domains from STDIN
   get-unique-ipv4-addresses: Creates a list of unique IPv4 addresses from STDIN
   get-domain-level: Creates list of unique domains of a given level from STDIN
   get-zonefile-domains: Obtains the list of domains from a zone file
   get-zonefile-nameservers: Obtains the list of nameservers from a zone file
";


# Pending: duckduckgo: get-duckduckgo, get-webcrawler-batch, get-webcrawler

use Socket();
use IPC::Open3;
use Net::IP;           # Normally libnet-ip-perl (was: libio-socket-ip-perl)
use Net::DNS;          # Normally libnet-dns-perl

$dns6revenum= "dnsrevenum6";
$wwwclient="";
$country_database="";
$dns_tld_database="";
$dns_suffix_database="";
$dns_dictionary_database="";
$rir_database="";
$wwwclient_sopt="";
$wwwclient_tout="";
$wwwclient_agnt="";
$MAXTIMEPERIOD=10;

sub ParseConfigFile{
	$configfile="/etc/ipv6toolkit.conf";

	open FILE, "$configfile" or die $!;

	while(<FILE>){
		if($_ =~ m/\s*WWW-client\s*=\s*([^\s]+)\s*/i){
			$wwwclient=$1;
		}
		elsif($_ =~ m/\s*Country-Database\s*=\s*([^\s]+)\s*/i){
			$country_database=$1;
		}
		elsif($_ =~ m/\s*DNS-Suffix-Database\s*=\s*([^\s]+)\s*/i){
			$dns_suffix_database=$1;
		}
		elsif($_ =~ m/\s*DNS-TLD-Database\s*=\s*([^\s]+)\s*/i){
			$dns_tld_database=$1;
		}
		elsif($_ =~ m/\s*DNS-Dictionary\s*=\s*([^\s]+)\s*/i){
			$dns_dictionary_database=$1;
		}
		elsif($_ =~ m/\s*RIR-Database\s*=\s*([^\s]+)\s*/i){
			$rir_database=$1;
		}
		elsif($_ =~ m/\s*DNS-UKGOV-Database\s*=\s*([^\s]+)\s*/i){
			$dns_ukgov_database=$1;
			# Typically at: https://assets.publishing.service.gov.uk/government/uploads/system/uploads/attachment_data/file/744721/List_of_.gov.uk_domain_names_as_at_1_October_2018.csv
		}
	}

	if($wwwclient=~ m/(?:\/[a-z-A-Z0-9-_]+\/)*curl/){
		$wwwclient_sopt=" -s ";
		$wwwclient_tout=" --max-time $MAXTIMEPERIOD ";
		$wwwclient_agnt=" --user-agent ";
		$wwwclient_addr=" --interface ";
	}
	elsif($wwwclient=~ m/(?:\/[a-z-A-Z0-9-_]+\/)*wget/){
		$wwwclient_sopt=" -q -O - ";
		$wwwclient_tout=" --timeout=$MAXTIMEPERIOD ";
		$wwwclient_agnt=" --user-agent=";
		$wwwclient_addr=" --bind-address=";
	}
	elsif($wwwclient=~ m/^\s*$/){
		print "ERROR: 'WWW-client' option not set in configuration file (/etc/ipv6toolkit.conf).\n";
		exit(1);
	}
	else{
		print "ERROR: WWW Client \"$wwwclient\" is not supported (please try 'curl' or 'wget').\n";
		exit(1);
	}

	close(FILE);
}

sub PrintConfigFile{
	print "WWW-client: $wwwclient\n";
	print "Country-Database=$country_database\n";
	print "DNS-TLD-Database=$dns_tld_database\n";
	print "DNS-Suffix-Database=$dns_suffix_database\n";
	print "RIR-Database=$rir_database\n";
	print "DNS-UKGOV-Database=$dns_ukgov_database\n";
	print "DNS-Dictionary=$dns_dictionary_database\n";
	print "[NOTE] WWW-client run as '$wwwclient$wwwclient_sopt\[$wwwclient_tout\]$wwwclient_agnt\"\[sample agent\]\"'\n";
}


sub SuffixForTLD{
		my $par= $_[0];
		chomp($par);
		my ($tld, $l)= split / /, $par;

		if(!defined($l)){
			$l= 0;
		}

		open FILE, "$dns_suffix_database" or die $!;

		$WAITINGFORBAR=1;
		$WAITINGFORDOMAIN=2;
		$WAITINGFORDATA=3;
		$PRINTINGDATA=4;
		$status=$WAITINGFORBAR;
		$printeddata=0;

		if($tld =~ m/([^.\s]+)$/){
			$ttld=$1;
		}

		while($fline=<FILE>){

			if($status == $WAITINGFORBAR){
				if($fline =~ m/^\/\//){
					$status=$WAITINGFORDOMAIN;
				}
			}
			elsif($status == $WAITINGFORDOMAIN){
				if($fline =~ m/^\/\//){
					next;
				}
				elsif($fline =~ m/^\s*$/){
					$status=$WAITINGFORBAR;
					next;
				}
				elsif($fline =~ m/^[-a-zA-Z0-9_.]+\s*$/){
					$status=$WAITINGFORDATA;

					if($fline =~ m/^\s*$ttld\s*$/i || $fline =~ m/^\s*$tld\s*$/i){
						$status=$PRINTINGDATA;
					}
					next;
				}
			}
			elsif($status == $WAITINGFORDATA){
				if($fline =~ m/^\s*$/){
					$status=$WAITINGFORBAR;
					next;
				}
			}
			elsif($status == $PRINTINGDATA){
				if($fline =~ m/^\/\//){
					next;
				}
				elsif($fline =~ m/^\s*$/){
					last;
				}
				elsif($l == 0 && $fline =~ m/^[\s.*]*([\S]+)$(?<=$tld$)/){
					$printeddata=1;
					print "$1\n";
				}
				elsif($l > 0 && $fline =~ m/^((?:[-a-zA-Z0-9_]+\.){$l}$tld)$(?<=$tld$)/){
					$printeddata=1;
					print "$1\n";
				}
			}
		}

		if(!$printeddata){
			print "$tld\n";
		}

		close(FILE);
}


sub Iso2ForCountry{
		my ($country) = $_[0];

		open FILE, "$country_database" or die $!;

		while(<FILE>){
			if($_ =~ m/"$country"\s*,\s*"([^"]+)"\s*/i){
				print "$1\n";
				close(FILE);
				last;
			}
		}

		close(FILE);
}


sub Iso3ForCountry{
		my ($country) = $_[0];

		open FILE, "$country_database" or die $!;

		while(<FILE>){
			if($_ =~ m/"$country"\s*,\s*"[^"]+"\s*,\s*"([^"]+)"\s*/i){
				print "$1\n";
				close(FILE);
				last;
			}
		}

		close(FILE);
}

sub Iso2ForRIR{
		my ($trir)= $_[0];

		open FILE, "$dns_tld_database" or die $!;

		while($line=<FILE>){
			
			if($line =~ m/\s*([^;]+);([^;]+);([^;]+);([^;\n\r]+)/){
				$country=$1;
				$iso2=lc($2);
				$iso3=$3;
				$rir=$4;

				if($rir=~ m/$trir/i){
					print "$iso2\n";
				}
			}
		} # while
}


sub Iso2ForRIROnline{
		my ($trir)= $_[0];

		$useragent=GetRandomUserAgent();

		do{
			@lines=`$wwwclient $wwwclient_sopt $wwwclient_agnt"$useragent" "https://www.nro.net/about-the-nro/list-of-country-codes-and-rirs-ordered-by-country-code/"`;

			$retcode= $?;
			my @domains;
			$start=0;

			if($retcode == 0){
				foreach $line (@lines){
					if($start == 0 && $line =~ m/\s*<td\s+class="cc-head">Region<\/td>/){
						$start=1;
					}
					elsif($start == 1){
						if(!$country){
							if($line=~/<td\s+class="cc-(?:white|grey)">([^<*]+)[*\s]*<\/td>/){
								$country=$1;
							}
						}
						elsif(!$iso2){
							if($line=~/<td\s+class="cc-(?:white|grey)">([^<*]+)[*\s]*<\/td>/){
								$iso2=$1;
							}
						}
						elsif(!$iso3){
							if($line=~/<td\s+class="cc-(?:white|grey)">([^<*]+)[*\s]*<\/td>/){
								$iso3=$1;
							}
						}
						elsif(!$rir){
							if($line=~/<td\s+class="cc-(?:white|grey)">([^<*]+)[*\s]*<\/td>/){
								$rir=$1;

								if($rir=~ m/$trir/i){
									print "$iso2\n";
								}

								undef $country;
								undef $iso2;
								undef $iso3;
								undef $rir;
							}
						}
					}

				} # foreach $line;
			}   # if $? == 0
		}while($retcode != 0);
}


sub TLDForRIR{
		my ($trir)= $_[0];

		open FILE, "$dns_tld_database" or die $!;

		while($line=<FILE>){
			
			if($line =~ m/\s*([^;]+);([^;]+);([^;]+);([^;\n\r]+)/){
				$country=$1;
				$iso2=lc($2);
				$iso3=$3;
				$rir=$4;

				if($rir=~ m/$trir/i){
					if($iso2 eq "GB" || $iso2 eq "gb"){
						print "uk\n";
					}
					else{
						print "$iso2\n";
					}
				}
			}
		} # while
}

sub TLDForRIROnline{
		my ($trir)= $_[0];

		$useragent=GetRandomUserAgent();

	do{
		@lines=`$wwwclient $wwwclient_sopt $wwwclient_agnt"$useragent" "https://www.nro.net/about-the-nro/list-of-country-codes-and-rirs-ordered-by-country-code/"`;

		my @domains;
		$start=0;
		$retcode=$?;

		if($retcode == 0){
			foreach $line (@lines){
				if($start == 0 && $line =~ m/\s*<td\s+class="cc-head">Region<\/td>/){
					$start=1;
				}
				elsif($start == 1){
					if(!$country){
						if($line=~/<td\s+class="cc-(?:white|grey)">([^<*]+)[*\s]*<\/td>/){
							$country=$1;
						}
					}
					elsif(!$iso2){
						if($line=~/<td\s+class="cc-(?:white|grey)">([^<*]+)[*\s]*<\/td>/){
							$iso2=lc($1);
						}
					}
					elsif(!$iso3){
						if($line=~/<td\s+class="cc-(?:white|grey)">([^<*]+)[*\s]*<\/td>/){
							$iso3=$1;
						}
					}
					elsif(!$rir){
						if($line=~/<td\s+class="cc-(?:white|grey)">([^<*]+)[*\s]*<\/td>/){
							$rir=$1;

							if($rir=~ m/$trir/i){
								if($iso2 eq "GB" || $iso2 eq "gb"){
									print "uk\n";
								}
								else{
									print "$iso2\n";
								}
							}

							undef $country;
							undef $iso2;
							undef $iso3;
							undef $rir;
						}
					}
				}

			} # foreach $line;
		}   # if $? == 0
	}while($retcode != 0);
}

sub CountryForContinent{
		my ($continent)= $_[0];

		open FILE, "$country_database" or die $!;

		while(<FILE>){
			if($_ =~ m/\s*"([^"]+)"\s*,\s*"[^"]+"\s*,\s*"[^"]+"\s*,\s*"[^"]+"\s*,\s*"[^"]+"\s*,\s*"[^"]+"\s*,\s*"[^"]+"\s*,\s*"[^"]+"\s*,\s*"[^"]+"\s*,\s*"$continent"\s*/i){
				print "$1\n";
			}
		}

		close(FILE);
}


sub GetASN{

	do{
		@revname=`addr6 -a $_[0] -r`;

		$retcode=$?;

		if($retcode == 0){
			chomp($revname[0]);
			$queryname= $revname[0] . ".origin6.asn.cymru.com.";
			@reverse=`host -t TXT $queryname`;

			if($reverse[0] =~ m/\"\s*((\d+)\s*)\s+|"/){
				return($2);
			}
		}
	}while($retcode != 0);

	return("");
}

@filetypesstr= ("doc", "pdf", "xls", "ppt", "txt", "pptx", "docx", "dot");



sub AddAddress{
	my($iface, $addr, $len, $os) = @_;

	@revname=`addr6 -A $addr/$len`;
	$randomizedaddress=  $revname[0];
        chomp($randomizedaddress);

	if($randomizedaddress =~ m/^inet_pton\(\)/){
		print "Invalid prefix: $addr/$len\n";
		exit(1); # XXX
	}

	if($os eq "linux"){
		@revname=`ip -6 addr add $randomizedaddress/64 dev $iface`;
	}
	elsif($os eq "bsd"){
		@revname=`ifconfig $iface inet6 $randomizedaddress/64 add`;
	}

	return($randomizedaddress);
}


sub DelAddress{
	my($iface, $addr, $len, $os) = @_;

	if($os eq "linux"){
		@revname=`ip -6 addr del $randomizedaddress/64 dev $iface`;
	}
	elsif($os eq "bsd"){
		@revname=`ifconfig $iface inet6 $randomizedaddress/64 delete`;
	}

}





# This function is employed to generate the list of domain sites that are excluded in one of the subphases
sub GetRemSitesString{
	my($dref, $searchdomain, $mode) = @_;

#	print "SEARCHDOMAIN: $searchdomain, MODE: $mode\n";

	$MAXREMOVEDSITES=25;

	$tsearchdomain=$searchdomain;


	$sddots=0; # Number of dots in search domain.

#	@remsites; # Contains domain name to be removed
#	@cdremsites # Contains number of dots in domain to be removed

	while($tsearchdomain =~ s/\.//){
		$sddots++;
	}	

#print "Dots in search: $sddots\n";

	# Go through list of all domains
	for($i=0; $i<= $#$dref; $i++){
		$tdots=0;  # Number of dots in current domain
		$tdomain= $dref->[$i];

		if($tdomain !~ m/$searchdomain(?::\d+)?$/){
			next;
#print "# When computing excludedomains, $tdomain didn't match (search: $searchdomain)\n";
		}
		else{
#print "# $tdomain is good candidate for exclusion list\n";
		}



		if($mode==0){
			while($tdomain =~ s/\.//){
				$tdots++;
			}

			$tdomain= $dref->[$i];
		}
		else{
			if($tdomain=~ m/www\.([-0-9a-zA-Z.]+)/){
				$tdomain=$1;
			}

			$stdomain=$tdomain;

			while($tdomain =~ s/\.//){
				$tdots++;
			}

			$tdomain=$stdomain;
		}

#print "Dots actual: $tdots, Dominio actual: $tdomain\n";

# Si la lista es menor q el max numero, lo inserto si o si.
# Voy del primero al ultimo, y meto en primero lugar el que es mayor.
# Si la lista esta completa, solo lo meto si la lngitud es mayor

		if($tdots <= $sddots){
			next;   # We don't remove domains of the same "depth"
		}

		for($j=0; $j <= $#remsites; $j++){
			if($tdomain eq $remsites[$j]){
				last;
			}
		}

		if($j <= $#remsites){
			next;
		}

		for($j=0; $j <= $#remsites; $j++){
#print "Comparing ($tdots, $tdomain) with ($cdremsites[$j], $remsites[$j])\n";
			if($tdots >= $cdremsites[$j]){
				$tempdomain= $remsites[$j];
				$tempdots= $cdremsites[$j];
				$remsites[$j]= $tdomain;
				$cdremsites[$j]= $tdots;
				$tdomain= $tempdomain;
				$tdots= $tempdots;
			}
		}

		if($j > $#remsites && $j <= ($MAXREMOVEDSITES-1)){
			$remsites[$j]= $tdomain;
			$cdremsites[$j]= $tdots;
		}
	}


	# Compute string
	$remsitesstring="";

	for($i=0; $i<=$#remsites; $i++){
#		print "DOTS: $cdremsites[$i], DOMAIN: $remsites[$i]\n";
		$remsitesstring= "$remsitesstring+-site%3a$remsites[$i]"
	}

	return($remsitesstring);
}


# Genarates list of domains to walk the tree
sub	GenDomListLevel{
	my($dref, $remref, $czone, $level, $limit) = @_;

	# Go through list of all domains
	for($w=0; $w<= $#$dref; $w++){
		if($limit > 0 && $#$remref >= ($limit - 1)){
			last; # Got as many as we needed
		}
		else{
			#print "NO LIMIT\n";
		}

		$tdomain= $dref->[$w];

		$lele=$level-1;

		if(	$dref->[$w]=~  m/www\.((?:[-0-9a-zA-Z]+\.){$lele}$czone)/){
			#print "#### EXCLUDED DOMAIN NUEVO $dref->[$w] (Starts with www) ####\n";
			next;
		}
		else{
			#print "# TODO BIEN!\n";
		}

		if(	$dref->[$w]=~  m/((?:[-0-9a-zA-Z]+\.){$level}$czone)(?::\d+)?/){
			$tdomain= $1;
		}
		else{
			next;
		}

		for($x=0; $x <= $#$remref; $x++){
			if($tdomain eq $remref->[$x]){
				last;
			}
		}

		if($x <= $#$remref){
			next;
		}
		else{
			$remref->[$#$remref + 1]= $tdomain;
		}
	}
}



sub GenRemListStr{
	my ($remref, $j)= @_;

	# Compute string
	$remsitesstring="";

	for($i=0; $i<=$#$remref && $i<=$j; $i++){
		$remsitesstring= "$remsitesstring+-site%3a$remref->[$i]"
	}

	return($remsitesstring);
}


sub GetRandomUserAgent{
	$uagent[0]="Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1";
	$uagent[1]="Mozilla/5.0 (Windows NT 6.3; rv:36.0) Gecko/20100101 Firefox/36.0";
	$uagent[2]="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10; rv:33.0) Gecko/20100101 Firefox/33.0";
	$uagent[3]="Mozilla/5.0 (X11; Linux i586; rv:31.0) Gecko/20100101 Firefox/31.0";
	$uagent[4]="Mozilla/5.0 (Windows NT 6.1; WOW64; rv:31.0) Gecko/20130401 Firefox/31.0";
	$uagent[5]="Mozilla/5.0 (Windows NT 5.1; rv:31.0) Gecko/20100101 Firefox/31.0";
	$uagent[6]="Mozilla/5.0 (Windows NT 6.1; WOW64; rv:29.0) Gecko/20120101 Firefox/29.0";
	$uagent[7]="Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gecko/20100101 Firefox/29.0";
	$uagent[8]="Mozilla/5.0 (X11; OpenBSD amd64; rv:28.0) Gecko/20100101 Firefox/28.0";
	$uagent[9]="Mozilla/5.0 (X11; Linux x86_64; rv:28.0) Gecko/20100101 Firefox/28.0";
	$uagent[10]="Mozilla/5.0 (Windows NT 6.1; rv:27.3) Gecko/20130101 Firefox/27.3";
	$uagent[11]="Mozilla/5.0 (Windows NT 6.2; Win64; x64; rv:27.0) Gecko/20121011 Firefox/27.0";
	$uagent[12]="Opera/9.80 (X11; Linux i686; Ubuntu/14.10) Presto/2.12.388 Version/12.16";
	$uagent[13]="Opera/9.80 (Windows NT 6.0) Presto/2.12.388 Version/12.14";
	$uagent[14]="Mozilla/5.0 (Windows NT 6.0; rv:2.0) Gecko/20100101 Firefox/4.0 Opera 12.14";
	$uagent[15]="Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0) Opera 12.14";
	$uagent[16]="Opera/12.80 (Windows NT 5.1; U; en) Presto/2.10.289 Version/12.02";
	$uagent[17]="Opera/9.80 (Windows NT 6.1; U; es-ES) Presto/2.9.181 Version/12.00";
	$uagent[18]="Opera/9.80 (Windows NT 5.1; U; zh-sg) Presto/2.9.181 Version/12.00";
	$uagent[19]="Opera/12.0(Windows NT 5.2;U;en)Presto/22.9.168 Version/12.00";
	$uagent[20]="Opera/12.0(Windows NT 5.1;U;en)Presto/22.9.168 Version/12.00";
	$uagent[21]="Mozilla/5.0 (Windows NT 5.1) Gecko/20100101 Firefox/14.0 Opera/12.0";
	$uagent[22]="Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36";
	$uagent[23]="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.1 Safari/537.36";
	$uagent[24]="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36";
	$uagent[25]="Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36";
	$uagent[26]="Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2226.0 Safari/537.36";
	$uagent[27]="Mozilla/5.0 (Windows NT 6.4; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36";
	$uagent[28]="Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36";
	$uagent[29]="Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2224.3 Safari/537.36";
	$uagent[30]="Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.93 Safari/537.36";
	$uagent[31]="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.124 Safari/537.36";
	$uagent[32]="Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36";
	$uagent[33]="Mozilla/5.0 (Windows NT 4.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36";
	$uagent[34]="Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36";
	$uagent[35]="Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36";
	$uagent[36]="Mozilla/5.0 (X11; OpenBSD i386) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.125 Safari/537.36";
	$uagent[37]="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1944.0 Safari/537.36";
	$uagent[38]="Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.3319.102 Safari/537.36";
	$uagent[39]="Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.2309.372 Safari/537.36";
	$uagent[40]="Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.2117.157 Safari/537.36";

	return ($uagent[rand($#uagent+1)%($#uagent+1)]);
}


sub SigIntHandler{
	if($gotaddress){
		DelAddress($iface, $gotaddress, $chpref, $chos);
	}

	exit 0;
}

sub GetDomainsFromCrt{
	my($pdomain, $pzone, $pmaxresults, $filehandle) = @_;
	$useragent=GetRandomUserAgent();

	# XXX Ugly hack to override the default (usually small) timeout value
	$wwwclient_tout=~ s/\d+/240/;


	$command="$wwwclient $wwwclient_sopt $wwwclient_agnt\"$useragent\" \"https://crt.sh/?q=%.$czone&output=json\"";
	$doagain_f=1;
	$times= 0;

	do{
		$pid = open3(\*CHLD_IN, \*CHLD_OUT, \*CHLD_ERR, $command, );
		close(CHLD_IN);
		close(CHLD_ERR);

		while($line=<CHLD_OUT>){

			# Check that we got a results page
			if($doagain_f && $line =~ m/^\[[^]]*]$/){
				$doagain_f=0;
			}

			$line =~ s/\\"/'/g; # Replaces escaped double-qutes with single-quotes

			while($line=~ s/{"issuer_ca_id":\d+\s*,\s*"issuer_name":"[^"]*"\s*,\s*"name_value":"([^"]+)","min_cert_id":\d+,"min_entry_timestamp":"[^"]+","not_before":"[^"]+","not_after":"[^"]+"}//){
				$grabbeddomain=$1;

				# Wildcards are not allwed in valid domains
				if($grabbeddomain=~ m/\*/){
					next;
				}

				# at signs are not allwed in valid domains
				if($grabbeddomain=~ m/@/){
					next;
				}

				# Remove "domains" that contain spaces
				if($grabbeddomain=~ m/ /){
					next;
				}

				if($grabbeddomain!~ m/$pzone/){
					next;
				}


				for($i=0; $i<= $#$pdomain; $i++){
					if($pdomain->[$i] eq $grabbeddomain){
						last;
					}
				}

				if($i > $#$pdomain){
					$pdomain->[$#$pdomain + 1]=$grabbeddomain;
					print $filehandle "$grabbeddomain\n";
				}
			}
		} # foreach $line;

		waitpid( $pid, 0 );

		if($doagain_f){
			$times++;
			sleep(10);
		}

	}while($doagain_f && $times <= 5);
}


sub GetDomainsFromBingDict{
#		GetDomainsFromCrawler(\@domain, $czone, $level, $maxresults);	
		my($pdomain, $pzone, $plevel, $pmaxresults, $dictfp, $intprotocol, $srcaddress) = @_;
		my $czone=$pzone;
		$lineno=0;

		$MAXREMOVEDSITES=25;

# Now set from the config file
#		$MAXTIMEPERIOD=10;

#		$maxtime=$MAXTIMEPERIOD;

#		if($maxtime < 5){
#			$maxtime=5;
#		}

		$MAXCNNORESULTS=125;
		$MINCNNORESULTS= 10;

		$limcnnoresults= $MAXCNNORESULTS/($plevel+1);

		if($limcnnoresults < $MINCNNORESULTS){
			$limcnnoresults= $MINCNNORESULTS;
		}

		$MAXCCOLLISIONS= 20;
		$MINCCOLISSIONS= 10;
		$limccollisions= $MAXCCOLLISIONS/($plevel+1);

		if($limccollisions < $MINCCOLISSIONS){
			$limccollisions= $MINCCOLISSIONS;
		}


		# This triggers a change in search string after cumulative number of pages that resulted
		# in 100% collisions
		$cphaseonesubone= $limccollisions/3;
		if($cphaseonesubone<4){
			$cphaseonesubone=4;
		}

		# This triggers changes in subphases that should change faster
		$csubphase= $limccollisions/4;
		if($csubphase < 10){
			$csubphase=10;
		}

		$MAXBLANKPAGE= 25;
		$MINBLANKPAGE= 5;

		$limblankpage= $MAXBLANKPAGE/($plevel+1);

		if($limblankpage < $MINBLANKPAGE){
			$limblankpage= $MINBLANKPAGE;
		}


		$maxresults= $pmaxresults;

		if($intprotocol == 4){
			$intprot="-4";
			$sengine="www.bing.com";
		}
		elsif($intprotocol == 6){
			$intprot="-6";
			$sengine="www2.bing.com";
		}
		else{
			$intprot="";
			$sengine="www.bing.com";
		}

		if($srcaddress eq "ANY"){
			$specaddr="";
		}
		else{
			# Safeguard -- should never happen

			if($intprotocol != 6){
				print "SPecified to bind IPv6 address, but not using IPv6\n";
				exit(1);
			}

			$specaddr="$wwwclient_addr$srcaddress";
		}


		$cur= 1;
		$nresults=0;
		$fpzeromatch=0;
		$nzeromatch=0;
		$czeromatch=0;
		$ncollisions=0;
		$ccollisions=0;
		$cnnoresults=0;
		$continue=1;
		$phase=0;
		$subphase=0;
		$heur="";
		$sitesstr="";
		$ftstr="";
		$remsites="";
		$totalqueries=0;

		# If we are doing mass search, we need a keyword to search for
		if($czone eq ""){
			$heur="+z";
		}

		do{   # Iteration for each page
			$matches=0;
			$totalmatches=0;
			$collisions=0;
			$hasnext=0;
			$noresults=0;
			$trydifferent=0;

			# --user-agent "Mozilla/4.0"  works fine most of the time. curl || wget original headers cause trouble.
			$useragent=GetRandomUserAgent();

			if($czone ne ""){
				$stringcommand="$wwwclient $intprot $specaddr $wwwclient_sopt $wwwclient_tout $wwwclient_agnt\"$useragent\" \"https://$sengine/search?q=site%3A$czone$sitesstr$heur&first=$cur\"";

				@revname=`$wwwclient $intprot $specaddr $wwwclient_sopt $wwwclient_tout $wwwclient_agnt"$useragent" "https://$sengine/search?q=site%3A$czone$sitesstr$heur&first=$cur"`;
			}
			else{
				$stringcommand="$wwwclient $intprot $specaddr $wwwclient_sopt $wwwclient_tout $wwwclient_agnt\"$useragent\" \"https://$sengine/search?q=$sitesstr$heur&first=$cur\"";

				@revname=`$wwwclient $intprot $specaddr $wwwclient_sopt $wwwclient_tout $wwwclient_agnt"$useragent" "https://$sengine/search?q=$sitesstr$heur&first=$cur"`;
			}

# print "Envie: $stringcommand\n";

			$totalqueries++;

			if($? == 0){
				foreach $line (@revname){
					if($line=~ m/<div class="sw_next">[^<]+<\/div><\/a>/){
						$hasnext=1;
					}

					if($line=~ m/<ol id="b_results" role="main" aria-label="[^"]+"><li class="b_no">\s*<h1>/){
						$noresults=1;
						$cnnoresults++;

#						if($cnnoresults >= $limcnnoresults){
#							$continue=0;
#						}

						# If got no results, try a different word
						$trydifferent=1;

					}

					while(!$trydifferent && $continue && $line =~ s/<h2[^>]*><a href="https?[^"\/]*\/\/([^"\/]*)//){
						$totalmatches++;
						$grabbeddomain= $1;


						if($czone ne "" && $grabbeddomain !~ m/$czone(:\d+)?/){
							#print "##### DEBUG: Discarded $grabbeddomain\n";
							next;
						}

						$matches++;
						$nzeromatch=0;
						$czeromatch=0;

						for($i=0; $i<= $#$pdomain; $i++){
							if($pdomain->[$i] eq $grabbeddomain){
								$collisions++;
								last;
							}
						}

						if($i > $#$pdomain){
							if($maxresults < 0 || ($maxresults >= 0 &&  $#$pdomain < $maxresults)){
								$pdomain->[$#$pdomain + 1]=$grabbeddomain;
								print "$grabbeddomain\n";
								$nresults++;
							}
							else{
								$continue=0;
							}
						}

						if($maxresults >= 0 && $#$pdomain >=$maxresults){
							$continue=0;
						} 
					}
				}

				#  There are a number of posible results:
				#
				# 1) We didn't get any results:
				#     a) because search engine banned us (condition: no matches, but no "no results" page)
				#        ACTION: Wait for some time, and retry with same query string and same "results page"
				#     b) Because the query string didn't yield results (condition: "no results" page)
				#        ACTION: First couple of times, wait a bit, and retry. Otherwise, change the query string. And
				#                if we hit the threshold of "no results", quit.
				# 2) We got results:
				#     a) Some were new results
				#        ACTION: Keep searching
				#     b) All were collisions
				#        ACTION: If we hit threshold, change query string. Otherwise, keep searching

				if($lineno == 0 && $czone eq ""){
					$trydifferent=1;
				}
				else{
					$trydifferent=0;
				}

				if($totalmatches == 0){
					#print "#### DEBUG: GOT NO MATCHES ####\n";
					if($noresults == 1){
						#print "#### DEBUG: GOT NO RESULTS ####\n";
						if($nzeromatch == 0){
							$fpzeromatch=$cur;  # Record "when" this happened
						}

						$nzeromatch++;

						# The continue = 0 condition was set above, anyway
						#if($cnnoresults > $limcnnoresults){
							# If we get "no results" after 25 pages we quit
							# print "#### Bing showed no results for our query, after retries. Quitting.... ####\n";
						#	$continue=0;
						#}
						#else{
							# Before we were trying the same query. But it does not really make sense.
						#	$trydifferent=1;
						#}

						# When working with dictionaries, it is expected that some wrods do not yield any results
						$trydifferent=1;
					}
					else{
						#print "#### DEBUG: GOT BLANK PAGE ####\n";

						# If we dn't even get a page that says "no results", we assume we have been banned. Back off, and retry.
						if($nzeromatch == 0){
							$fpzeromatch=$cur;  # Record "when" this happened
							#print "# Offset: $fpzeromatch\n";
						}

						$nzeromatch++;
						$czeromatch++;

						# If it is the first time the main regex fails, chances are that bing has "banned" us. Wait a bit, and retry.
						#print "#### DEBUG: SLEEP A BIT ####\n";
						sleep(rand(($nzeromatch)*15));

						#if($nzeromatch > $limblankpage || $czeromatch  > $limblankpage){
							# If we get nothing after 25 pages we quit
							#print "#### Bing has banned us. Quitting.... ####\n";
							#$continue=0;
						#}

						if($nzeromatch > $limblankpage){
							# If we get nothing after 25 pages we quit
							$trydifferent=1;
						}

						if($plevel > 0 && $nzeromatch > 2){
							$trydifferent=1;
						}
					}
				}
				else{
					# We did get search results

					# If all were collisions, count the number of collisions to exit on threshold.
					if($collisions >= $matches){
						#print "#### DEBUG: GOT ALL COLLISIONS ####\n";
						$ncollisions++;
						$ccollisions++;  # Cumulative colissions even across changes in search string

						if($ncollisions > $csubphase){
							$trydifferent=1;
						}

						$cur= $cur+ $totalmatches;
					}
					else{
						$difmatch=$totalmatches-$matches;
						# Not all results were collisions. So get the next search results.
						$cur= $cur+$totalmatches;
						$ncollisions=0;
						$ccollisions=0;
						#print "### GOT NEW DATA ####\n"
					}

					# If last search didn't have "Next results", we have no other option than changing the query stringg
					if($hasnext == 0){
						#print "#### DEBUG: NO NEXT RESULTS ####\n";
						$trydifferent=1;
					}

				}  # if/else on whether there were matches

				if($trydifferent == 1){
					# subfase0: no pongo nada extra
					# subfase1: tipos de archivos
					# subfase2: elimino sites
					# subfase3: elimino sites sin www;
					# subfase4: tipos de archivo + elimino sites	
					# subfase5: tipos de archivo ´+ sites sin www

					# XXX
					$lastkeyword=$dictkeyword;
					$dictkeyword="";
					while($dictword=<$dictfp>){
						$lineno++;
						chomp($dictword);

						if($dictword =~ m/^\s*#/){
							next;
						}

						$dictword=~ s/\r//g;


#print "# Lei: $dictword";
#						if( $dictword =~ m/([^\/0-9\s]+[^\/0-9]*)/){
						if( $dictword =~ m/([^\/0-9\n]+)/){
							$dictkeyword=$1;
							$dictkeyword=~ s/ /%20/g;
#print "# Palabra leida: $dictkeyword\n";
							last;
						}
					}

					if(!$dictword){
#print "# Error al leer palabra\n";
						print "# Finished dictionary! (last keyword: $lastkeyword, line: $lineno)\n";
						$continue=0;
					}
					else{
#print "# Palabra seleccionada: $dictkeyword\n";
					}

					$heur="+$dictkeyword";
					$cur=1;

					$trydifferent=0;
				}  # End of "trydifferent"
			}  # End of $? (result of wget)
			else{
				#print "# \$? was different from 0 (\$?== $?, command: '$stringcommand')\n";
				$totalqueries--;
			}

		}while($continue == 1);
#	was: }while($? == 0 && $continue == 1);

#		if($continue != 1){
#			print "# \$continue was different from 1 (\$continue== $continue)\n";
#		}

		if($maxresults > 0 && $#$pdomain >= $maxresults){
			return;
		}


		# We will now exclude next level domains

		undef @remsites;

		GenDomListLevel($pdomain, \@remsites, $czone, 1, $MAXREMOVEDSITES);

		$continue=1;

		for($jjj= 0; $jjj <= $#remsites && $continue == 1; $jjj++){
			$cur= 1;

			$sitesstr= GenRemListStr(\@remsites, $jjj);
			$trydifferent=0;

			do{   # Iteration for each page
				$matches=0;
				$totalmatches=0;
				$collisions=0;
				$hasnext=0;
				$noresults=0;

				$useragent=GetRandomUserAgent();

				if($czone ne ""){
					$stringcommand="$wwwclient $wwwclient_sopt $wwwclient_tout $wwwclient_agnt\"$useragent\" \"https://www.bing.com/search?q=site%3A$czone$sitesstr&first=$cur\"";

					@revname=`$wwwclient $wwwclient_sopt $wwwclient_tout $wwwclient_agnt"$useragent" "https://www.bing.com/search?q=site%3A$czone$sitesstr&first=$cur"`;
				}
				else{
					$stringcommand="$wwwclient $wwwclient_sopt $wwwclient_tout $wwwclient_agnt\"$useragent\" \"https://www.bing.com/search?q=$sitesstr&first=$cur\"";

					@revname=`$wwwclient $wwwclient_sopt $wwwclient_tout $wwwclient_agnt"$useragent" "https://www.bing.com/search?q=$sitesstr&first=$cur"`;
				}

				$totalqueries++;

				if($? == 0){
					foreach $line (@revname){
						if($continue == 0){
							last;
						}

						if($line=~ m/<div class="sw_next">[^<]+<\/div><\/a>/){
							$hasnext=1;
						}

						if($line=~ m/<ol id="b_results" role="main" aria-label="[^"]+"><li class="b_no">\s*<h1>/){
							$noresults=1;
						}

						while($continue && $line =~ s/<h2[^>]*><a href="https?[^"\/]*\/\/([^"\/]*)//){
							$totalmatches++;
							$grabbeddomain= $1;

							if($czone ne "" && $grabbeddomain !~ m/$czone(:\d+)?/){
								next;
							}

							$matches++;
							$nzeromatch=0;

							for($k=0; $k<= $#$pdomain; $k++){
								if($pdomain->[$k] eq $grabbeddomain){
									$collisions++;
									last;
								}
							}


							if($k > $#$pdomain){
								if($maxresults < 0 || ($maxresults >= 0 &&  $#$pdomain < $maxresults)){
									$pdomain->[$#$pdomain + 1]=$grabbeddomain;
									print "$grabbeddomain\n";
									$nresults++;
								}
								else{
									$continue=0;
								}
							}

							if($maxresults >= 0 && $#$pdomain>=$maxresults){
								$continue=0;
							} 
						} # while() that reads lines
					}  # foreach $line

					$trydifferent=0;

					if($totalmatches == 0){
						if($noresults == 1){
							$trydifferent=1;
						}
						else{
							#print "#### DEBUG: GOT BLANK PAGE ####\n";
							$nzeromatch++;

							# If it is the first time the main regex fails, chances are that bing has "banned" us. Wait a bit, and retry.
							#print "#### DEBUG: SLEEP A BIT ####\n";
							sleep(rand(($nzeromatch)*20));

							if($nzeromatch > 5){
								#	print "#### Bing has banned us. Quitting.... ####\n";
								$trydifferent=1;
							}
						}
					}
					else{
						# We did get search results

						# If all were collisions, count the number of collisions to exit on threshold.
						if($collisions >= $matches){
							#print "#### DEBUG: GOT ALL COLLISIONS ####\n";
							$ncollisions++;

							if($ncollisions >= 10){
								$trydifferent=1;
							} 

							$cur= $cur+ $totalmatches;
						}
						else{
							$difmatch=$totalmatches-$matches;
							# Not all results were collisions. So get the next search results.
							$cur= $cur+$totalmatches;
							$ncollisions=0;
						}

						# If last search didn't have "Next results", we have no other option than changing the query stringg
						if($hasnext == 0){
							#print "#### DEBUG: NO NEXT RESULTS ####\n";
							$trydifferent=1;
						}
					}  # if/else on whether there were matches
				}  # End of $? (result of wget)
				else{
					# print "# \$? was different from 0 (\$?== $?, command: '$stringcommand')\n";
					$totalqueries--;
				}
#			}while($? == 0 && $trydifferent == 0);
			}while($trydifferent == 0);

			if($#remsites < ($MAXREMOVEDSITES-1)){
				# If list is not complete, attempt to complete it
				GenDomListLevel($pdomain, \@remsites, $czone, 1, $MAXREMOVEDSITES);
			}
		} # for that adds new removed sites;
}



sub GetDomainsFromBing{
#		GetDomainsFromCrawler(\@domain, $czone, $level, $maxresults, protocol);	
		my($pdomain, $pzone, $plevel, $pmaxresults, $intprotocol) = @_;
		my $czone=$pzone;

		$MAXREMOVEDSITES=25;

# Now set from the config file
#		$MAXTIMEPERIOD=10;

#		$maxtime=$MAXTIMEPERIOD;

#		if($maxtime < 5){
#			$maxtime=5;
#		}

		$MAXCNNORESULTS=125;
		$MINCNNORESULTS= 10;

		$limcnnoresults= $MAXCNNORESULTS/($plevel+1);

		if($limcnnoresults < $MINCNNORESULTS){
			$limcnnoresults= $MINCNNORESULTS;
		}

		$MAXCCOLLISIONS=150;
		$MINCCOLISSIONS= 10;
		$limccollisions= $MAXCCOLLISIONS/($plevel+1);

		if($limccollisions < $MINCCOLISSIONS){
			$limccollisions= $MINCCOLISSIONS;
		}


		# This triggers a change in search string after cumulative number of pages that resulted
		# in 100% collisions
		$cphaseonesubone= $limccollisions/3;
		if($cphaseonesubone<4){
			$cphaseonesubone=4;
		}

		# This triggers changes in subphases that should change faster
		$csubphase= $limccollisions/4;
		if($csubphase < 4){
			$csubphase=4;
		}

		$MAXBLANKPAGE= 25;
		$MINBLANKPAGE= 5;

		$limblankpage= $MAXBLANKPAGE/($plevel+1);

		if($limblankpage < $MINBLANKPAGE){
			$limblankpage= $MINBLANKPAGE;
		}


		$maxresults= $pmaxresults;


		if($intprotocol == 4){
			$intprot="-4";
			$sengine="www.bing.com";
		}
		elsif($intprotocol == 6){
			$intprot="-6";
			$sengine="www2.bing.com";
		}
		else{
			$intprot="";
			$sengine="www.bing.com";
		}


		$cur= 1;
		$nresults=0;
		$fpzeromatch=0;
		$nzeromatch=0;
		$czeromatch=0;
		$ncollisions=0;
		$ccollisions=0;
		$cnnoresults=0;
		$continue=1;
		$phase=0;
		$subphase=0;

		if($czone ne ""){
			$heur="";
		}
		else{
			$letterkey="a";
			$heur="+$letterkey";
		}

		$sitesstr="";
		$ftstr="";
		$remsites="";
		$totalqueries=0;

		do{   # Iteration for each page
			$matches=0;
			$totalmatches=0;
			$collisions=0;
			$hasnext=0;
			$noresults=0;

			# --user-agent "Mozilla/4.0"  works fine most of the time. curl || wget original headers cause trouble.
			$useragent=GetRandomUserAgent();

			if($czone ne ""){
				$stringcommand="$wwwclient $intprot $wwwclient_sopt $wwwclient_tout $wwwclient_agnt\"$useragent\" \"https://$sengine/search?q=site%3A$czone$sitesstr$ftstr$heur&first=$cur\"";
				@revname=`$wwwclient $intprot $wwwclient_sopt $wwwclient_tout $wwwclient_agnt"$useragent" "https://$sengine/search?q=site%3A$czone$sitesstr$ftstr$heur&first=$cur"`;
			}
			else{
				$stringcommand="$wwwclient $intprot $wwwclient_sopt $wwwclient_tout $wwwclient_agnt\"$useragent\" \"https://$sengine/search?q=$sitesstr$ftstr$heur&first=$cur\"";
				@revname=`$wwwclient $intprot $wwwclient_sopt $wwwclient_tout $wwwclient_agnt"$useragent" "https://$sengine/search?q=$sitesstr$ftstr$heur&first=$cur"`;
			}

			$totalqueries++;

			if($? == 0){
				foreach $line (@revname){
					if($line=~ m/<div class="sw_next">[^<]+<\/div><\/a>/){
						$hasnext=1;
					}

					if($line=~ m/<ol id="b_results" role="main" aria-label="[^"]+"><li class="b_no">\s*<h1>/){
						$noresults=1;
						$cnnoresults++;

						if($cnnoresults >= $limcnnoresults){
							$continue=0;
						}
					}

					while($continue && $line =~ s/<h2[^>]*><a href="https?[^"\/]*\/\/([^"\/]*)//){
						$totalmatches++;
						$grabbeddomain= $1;


						if($czone ne "" && $grabbeddomain !~ m/$czone(:\d+)?/){
							#print "##### DEBUG: Discarded $grabbeddomain\n";
							next;
						}

						$matches++;
						$nzeromatch=0;
						$czeromatch=0;

						for($i=0; $i<= $#$pdomain; $i++){
							if($pdomain->[$i] eq $grabbeddomain){
								$collisions++;
								last;
							}
						}

						if($i > $#$pdomain){
							if($maxresults < 0 || ($maxresults >= 0 &&  $#$pdomain < $maxresults)){
								$pdomain->[$#$pdomain + 1]=$grabbeddomain;
								print "$grabbeddomain\n";
								$nresults++;
							}
							else{
								$continue=0;
							}
						}

						if($maxresults >= 0 && $#$pdomain >=$maxresults){
							$continue=0;
						} 
					}
				}

				#  There are a number of posible results:
				#
				# 1) We didn't get any results:
				#     a) because search engine banned us (condition: no matches, but no "no results" page)
				#        ACTION: Wait for some time, and retry with same query string and same "results page"
				#     b) Because the query string didn't yield results (condition: "no results" page)
				#        ACTION: First couple of times, wait a bit, and retry. Otherwise, change the query string. And
				#                if we hit the threshold of "no results", quit.
				# 2) We got results:
				#     a) Some were new results
				#        ACTION: Keep searching
				#     b) All were collisions
				#        ACTION: If we hit threshold, change query string. Otherwise, keep searching

				$trydifferent=0;

				if($totalmatches == 0){
					#print "#### DEBUG: GOT NO MATCHES ####\n";
					if($noresults == 1){
						#print "#### DEBUG: GOT NO RESULTS ####\n";
						if($nzeromatch == 0){
							$fpzeromatch=$cur;  # Record "when" this happened
						}

						$nzeromatch++;

						# The continue = 0 condition was set above, anyway
						if($cnnoresults > $limcnnoresults){
							# If we get "no results" after 25 pages we quit
							# print "#### Bing showed no results for our query, after retries. Quitting.... ####\n";
							$continue=0;
						}
						else{
							# Before we were trying the same query. But it does not really make sense.
							$trydifferent=1;
						}
					}
					else{
						#print "#### DEBUG: GOT BLANK PAGE ####\n";

						# If we dn't even get a page that says "no results", we assume we have been banned. Back off, and retry.
						if($nzeromatch == 0){
							$fpzeromatch=$cur;  # Record "when" this happened
							#print "# Offset: $fpzeromatch\n";
						}

						$nzeromatch++;
						$czeromatch++;

						# If it is the first time the main regex fails, chances are that bing has "banned" us. Wait a bit, and retry.
						#print "#### DEBUG: SLEEP A BIT ####\n";
						sleep(rand(($nzeromatch)*15));

						if($nzeromatch > $limblankpage || $czeromatch  > $limblankpage){
							# If we get nothing after 25 pages we quit
							#print "#### Bing has banned us. Quitting.... ####\n";
							$continue=0;
						}

						if($plevel > 0 && $nzeromatch > 2){
							$trydifferent=1;
						}
					}
				}
				else{
					# We did get search results

					# If all were collisions, count the number of collisions to exit on threshold.
					if($collisions >= $matches){
						#print "#### DEBUG: GOT ALL COLLISIONS ####\n";
						$ncollisions++;
						$ccollisions++;  # Cumulative colissions even across changes in search string

						if( ($phase >= 2 && $ccollisions > $limccollisions) || ($plevel > 0 && $phase > 0 && $ccollisions > $limccollisions)){
							# If we get nothing after 100 pages (across change of query strings), and we are in 
							# phase 2 or more, we quit
							# print "#### Couldn't get any new results. Quitting.... ####\n";
							$continue=0;
						}

						if($phase == 0){
							if($subphase == 0){
								if($ncollisions > $cphaseonesubone){
									$trydifferent=1;
								}
							}
							else{
								if($ncollisions > $csubphase){
									$trydifferent=1;
								}								
							}
						}
						elsif($phase == 1){
							if($ncollisions > $cphaseonesubone){
								$trydifferent=1;
							}
						}
						elsif($phase == 2){
							if($ncollisions > $csubphase){
								$trydifferent=1;
							}
						}
						else{
							if($ncollisions > $csubphase){
								$trydifferent=1;
							}
						}

						$cur= $cur+ $totalmatches;
					}
					else{
						$difmatch=$totalmatches-$matches;
						# Not all results were collisions. So get the next search results.
						$cur= $cur+$totalmatches;
						$ncollisions=0;
						$ccollisions=0;
						#print "### GOT NEW DATA ####\n"
					}

					# If last search didn't have "Next results", we have no other option than changing the query stringg
					if($hasnext == 0){
						#print "#### DEBUG: NO NEXT RESULTS ####\n";
						$trydifferent=1;
					}

				}  # if/else on whether there were matches

				if($trydifferent == 1){
					# subfase0: no pongo nada extra
					# subfase1: tipos de archivos
					# subfase2: elimino sites
					# subfase3: elimino sites sin www;
					# subfase4: tipos de archivo + elimino sites	
					# subfase5: tipos de archivo ´+ sites sin www

					if($subphase == 0){
						$subphase=1;
						$ftindex=0;
						$ftstr="+filetype%3A$filetypesstr[$ftindex]";
						$ncollisions=0;
						$cur=1; # Since we change the search keys, we retrieve the first page of results
					}
					elsif($subphase == 1){
						if($ftindex >= $#filetypesstr){
							$ftstr="";
							$ftindex=0;
							$subphase=2;
							undef @remsites;
							$sitesstr=GetRemSitesString($pdomain, $czone, 0);
						}
						else{
							$ftindex++;
							$ftstr="+filetype%3A$filetypesstr[$ftindex]";
						}

						$ncollisions=0;
						$cur=1; # Since we change the search keys, we retrieve the first page of results
					}
					elsif($subphase == 2){
						$subphase=3;
						undef @remsites;
						$sitesstr=GetRemSitesString($pdomain, $czone, 1);
						$ncollisions=0;
						$cur=1; # Since we change the search keys, we retrieve the first page of results
					}
					elsif($subphase == 3){
						$subphase=4;
						undef @remsites;
						$sitesstr=GetRemSitesString($pdomain, $czone, 0);
						$ftindex=0;
						$ftstr="+filetype%3A$filetypesstr[$ftindex]";
						$ncollisions=0;
						$cur=1; # Since we change the search keys, we retrieve the first page of results
					}
					elsif($subphase == 4){
						if($ftindex >= $#filetypesstr){
							$subphase=5;
							$ftindex=0;
							$ftstr="+filetype%3A$filetypesstr[$ftindex]";
							undef @remsites;
							$sitesstr=GetRemSitesString($pdomain, $czone, 1);
						}
						else{
							$ftindex++;
							$ftstr="+filetype%3A$filetypesstr[$ftindex]";
						}

						$ncollisions=0;
						$cur=1; # Since we change the search keys, we retrieve the first page of results
					}
					elsif($subphase ==5){
						if($ftindex < $#filetypesstr){
							$ftindex++;
							$ftstr="+filetype%3A$filetypesstr[$ftindex]";
							$ncollisions=0;
							$cur=1; # Since we change the search keys, we retrieve the first page of results
						}
						else{
							$ftstr="";
							$ftindex=0;
							$sitesstr="";
							$subphase=0;

							if($phase == 0){
								# print "#### Phase 1  ####\n";
								$phase=1;
								$letterkey="a";
								$heur="+$letterkey";
								$ncollisions=0;
								$cur=1; # Since we change the search keys, we retrieve the first page of results
							}
							elsif($phase == 1){
								# In phase 1 we search for the domain name and one letter
								if($letterkey eq "z"){
									$letterkey="a";
									$numkey="0";
									$heur="+$numkey";
									$ncollisions=0;
									$phase=2;
									$cur=1; # Since we change the search keys, we retrieve the first page of results
									# print "#### Phase 2  ####\n";
								}
								else{
									$letterkey++;
									$ncollisions=0;
									$heur="+$letterkey";
									$cur=1; # Since we change the search keys, we retrieve the first page of results
								}
							}
							elsif($phase == 2){
								# In phase 2 we search for the domain name and a number

								if($numkey >= 10001){
									$letterkey="a";
									$numkey=0;
									$heur="+$letterkey+$numkey";
									$ncollisions=0;
									$phase=3;
									$cur=1; # Since we change the search keys, we retrieve the first page of results
									# print "#### Phase 3  ####\n";
								}
								else{
									$numkey++;
									$ncollisions=0;
									$cur=1; # Since we change the search keys, we retrieve the first page of results
									$heur="+$numkey";
								}
							}
							else{
								# In phase #3, we search for the domainname, plus a letter and a number
								if($numkey > 10000){
									if($letterkey eq "z"){
										# print "#### Tried my best. Quitting.... ####\n";
										$continue=0;
									}
									else{
										$letterkey++;
										$numkey=0;
										$cur=1; # Since we change the search keys, we retrieve the first page of results
										$heur="+$letterkey+$numkey";
										$ncollisions=0;
									}
								}
								else{
									$numkey++;
									$ncollisions=0;
									$cur=1; # Since we change the search keys, we retrieve the first page of results
									$heur="+$letterkey+$numkey";
								}
							} #end oh phase 
						} # end of else in subphase 5

					} #end of subphase

				}  # End of "trydifferent"
			}  # End of $? (result of wget)
			else{
#				print "# \$? was different from 0 (\$?== $?, command: '$stringcommand')\n";
				$totalqueries--;
			}
#		}while($? == 0 && $continue == 1);
		}while($continue == 1);

		if($maxresults > 0 && $#$pdomain >= $maxresults){
			return;
		}


		# We will now exclude next level domains

		undef @remsites;

		GenDomListLevel($pdomain, \@remsites, $czone, 1, $MAXREMOVEDSITES);

		$continue=1;

		for($jjj= 0; $jjj <= $#remsites && $continue == 1; $jjj++){
			$cur= 1;

			$sitesstr= GenRemListStr(\@remsites, $jjj);
			$trydifferent=0;

			do{   # Iteration for each page
				$matches=0;
				$totalmatches=0;
				$collisions=0;
				$hasnext=0;
				$noresults=0;

				$useragent=GetRandomUserAgent();

				if($czone ne ""){
					$stringcommand="$wwwclient $wwwclient_sopt $wwwclient_tout $wwwclient_agnt\"$useragent\" \"https://www.bing.com/search?q=site%3A$czone$sitesstr&first=$cur\"";
					@revname=`$wwwclient $wwwclient_sopt $wwwclient_tout $wwwclient_agnt"$useragent" "https://www.bing.com/search?q=site%3A$czone$sitesstr&first=$cur"`;
				}
				else{
					$stringcommand="$wwwclient $wwwclient_sopt $wwwclient_tout $wwwclient_agnt\"$useragent\" \"https://www.bing.com/search?q=$sitesstr&first=$cur\"";
					@revname=`$wwwclient $wwwclient_sopt $wwwclient_tout $wwwclient_agnt"$useragent" "https://www.bing.com/search?q=$sitesstr&first=$cur"`;
				}

				$totalqueries++;

				if($? == 0){
					foreach $line (@revname){
						if($continue == 0){
							last;
						}

						if($line=~ m/<div class="sw_next">[^<]+<\/div><\/a>/){
							$hasnext=1;
						}

						if($line=~ m/<ol id="b_results" role="main" aria-label="[^"]+"><li class="b_no">\s*<h1>/){
							$noresults=1;
						}

						while($continue && $line =~ s/<h2[^>]*><a href="https?[^"\/]*\/\/([^"\/]*)//){
							$totalmatches++;
							$grabbeddomain= $1;

							if($czone ne "" && $grabbeddomain !~ m/$czone(:\d+)?/){
								next;
							}

							$matches++;
							$nzeromatch=0;

							for($k=0; $k<= $#$pdomain; $k++){
								if($pdomain->[$k] eq $grabbeddomain){
									$collisions++;
									last;
								}
							}


							if($k > $#$pdomain){
								if($maxresults < 0 || ($maxresults >= 0 &&  $#$pdomain < $maxresults)){
									$pdomain->[$#$pdomain + 1]=$grabbeddomain;
									print "$grabbeddomain\n";
									$nresults++;
								}
								else{
									$continue=0;
								}
							}

							if($maxresults >= 0 && $#$pdomain>=$maxresults){
								$continue=0;
							} 
						} # while() that reads lines
					}  # foreach $line

					$trydifferent=0;

					if($totalmatches == 0){
						if($noresults == 1){
							$trydifferent=1;
						}
						else{
							#print "#### DEBUG: GOT BLANK PAGE ####\n";
							$nzeromatch++;

							# If it is the first time the main regex fails, chances are that bing has "banned" us. Wait a bit, and retry.
							#print "#### DEBUG: SLEEP A BIT ####\n";
							sleep(rand(($nzeromatch)*20));

							if($nzeromatch > 5){
								#	print "#### Bing has banned us. Quitting.... ####\n";
								$trydifferent=1;
							}
						}
					}
					else{
						# We did get search results

						# If all were collisions, count the number of collisions to exit on threshold.
						if($collisions >= $matches){
							#print "#### DEBUG: GOT ALL COLLISIONS ####\n";
							$ncollisions++;

							if($ncollisions >= 10){
								$trydifferent=1;
							} 

							$cur= $cur+ $totalmatches;
						}
						else{
							$difmatch=$totalmatches-$matches;
							# Not all results were collisions. So get the next search results.
							$cur= $cur+$totalmatches;
							$ncollisions=0;
						}

						# If last search didn't have "Next results", we have no other option than changing the query stringg
						if($hasnext == 0){
							#print "#### DEBUG: NO NEXT RESULTS ####\n";
							$trydifferent=1;
						}
					}  # if/else on whether there were matches
				}  # End of $? (result of wget)
				else{
#					print "# \$? was different from 0 (\$?== $?, command: '$stringcommand')\n";
					$totalqueries--;
				}

			}while($trydifferent == 0);

			if($#remsites < ($MAXREMOVEDSITES-1)){
				# If list is not complete, attempt to complete it
				GenDomListLevel($pdomain, \@remsites, $czone, 1, $MAXREMOVEDSITES);
			}
		} # for that adds new removed sites;
}




sub GetDomainsFromBingByAddr{
#		GetDomainsFromCrawler(\@domain, $czone, $level, $maxresults);	
		my($pdomain, $pzone, $pmaxresults, $intprotocol) = @_;
		my $czone=$pzone;

		$plevel=1;
		$MAXREMOVEDSITES=25;

# Now set from the config file
#		$MAXTIMEPERIOD=10;

#		$maxtime=$MAXTIMEPERIOD;

#		if($maxtime < 5){
#			$maxtime=5;
#		}

		$MAXCNNORESULTS=125;
		$MINCNNORESULTS= 10;

		$limcnnoresults= $MAXCNNORESULTS/($plevel+1);

		if($limcnnoresults < $MINCNNORESULTS){
			$limcnnoresults= $MINCNNORESULTS;
		}

		$MAXCCOLLISIONS=120;
		$MINCCOLISSIONS= 10;
		$limccollisions= $MAXCCOLLISIONS/($plevel+1);

		if($limccollisions < $MINCCOLISSIONS){
			$limccollisions= $MINCCOLISSIONS;
		}


		# This triggers a change in search string after cumulative number of pages that resulted
		# in 100% collisions
		$cphaseonesubone= $limccollisions/5;
		if($cphaseonesubone<2){
			$cphaseonesubone=2;
		}

		# This triggers changes in subphases that should change faster
		$csubphase= $limccollisions/5;
		if($csubphase < 2){
			$csubphase=2;
		}

		$MAXBLANKPAGE= 25;
		$MINBLANKPAGE= 5;

		$limblankpage= $MAXBLANKPAGE/($plevel+1);

		if($limblankpage < $MINBLANKPAGE){
			$limblankpage= $MINBLANKPAGE;
		}


		$maxresults= $pmaxresults;


		if($intprotocol == 4){
			$intprot="-4";
			$sengine="www.bing.com";
		}
		elsif($intprotocol == 6){
			$intprot="-6";
			$sengine="www2.bing.com";
		}
		else{
			$intprot="";
			$sengine="www.bing.com";
		}


		$cur= 1;
		$nresults=0;
		$fpzeromatch=0;
		$nzeromatch=0;
		$czeromatch=0;
		$ncollisions=0;
		$ccollisions=0;
		$cnnoresults=0;
		$continue=1;
		$phase=0;
		$subphase=0;
		$heur="";

		$letterkey="a";
		$heur="+$letterkey";


		$sitesstr="";
		$ftstr="";
		$remsites="";
		$totalqueries=0;

		do{   # Iteration for each page
			$matches=0;
			$totalmatches=0;
			$collisions=0;
			$hasnext=0;
			$noresults=0;

			# --user-agent "Mozilla/4.0"  works fine most of the time. curl || wget original headers cause trouble.
			$useragent=GetRandomUserAgent();

			if($cur == 1){
						$stringcommand="$wwwclient $intprot $wwwclient_sopt $wwwclient_tout $wwwclient_agnt\"$useragent\" \"https://$sengine/search?q=ip%3A$czone$sitesstr$ftstr$heur\"";
						@revname=`$wwwclient $intprot $wwwclient_sopt $wwwclient_tout $wwwclient_agnt"$useragent" "https://$sengine/search?q=ip%3A$czone$sitesstr$ftstr$heur"`;
			}
			else{

						$stringcommand="$wwwclient $intprot $wwwclient_sopt $wwwclient_tout $wwwclient_agnt\"$useragent\" \"https://$sengine/search?q=ip%3A$czone$sitesstr$ftstr$heur&first=$cur\"";
						@revname=`$wwwclient $intprot $wwwclient_sopt $wwwclient_tout $wwwclient_agnt"$useragent" "https://$sengine/search?q=ip%3A$czone$sitesstr$ftstr$heur&first=$cur"`;
			}
			$totalqueries++;

			if($? == 0){
				foreach $line (@revname){
					if($line=~ m/<div class="sw_next">[^<]+<\/div><\/a>/){
						$hasnext=1;
					}

					if($line=~ m/<ol id="b_results" role="main" aria-label="[^"]+"><li class="b_no">\s*<h1>/){
						$noresults=1;
						$cnnoresults++;

						if($cnnoresults >= $limcnnoresults){
							$continue=0;
						}
					}

					while($continue && $line =~ s/<h2[^>]*><a href="https?[^"\/]*\/\/([^"\/]*)//){
						$totalmatches++;
						$grabbeddomain= $1;


						$matches++;
						$nzeromatch=0;
						$czeromatch=0;

						for($i=0; $i<= $#$pdomain; $i++){
							if($pdomain->[$i] eq $grabbeddomain){
								$collisions++;
								last;
							}
						}

						if($i > $#$pdomain){
							if($maxresults < 0 || ($maxresults >= 0 &&  $#$pdomain < $maxresults)){
								$pdomain->[$#$pdomain + 1]=$grabbeddomain;
								print "$grabbeddomain\n";
								$nresults++;
							}
							else{
								$continue=0;
							}
						}

						if($maxresults >= 0 && $#$pdomain >=$maxresults){
							$continue=0;
						} 
					}
				}

				#  There are a number of posible results:
				#
				# 1) We didn't get any results:
				#     a) because search engine banned us (condition: no matches, but no "no results" page)
				#        ACTION: Wait for some time, and retry with same query string and same "results page"
				#     b) Because the query string didn't yield results (condition: "no results" page)
				#        ACTION: First couple of times, wait a bit, and retry. Otherwise, change the query string. And
				#                if we hit the threshold of "no results", quit.
				# 2) We got results:
				#     a) Some were new results
				#        ACTION: Keep searching
				#     b) All were collisions
				#        ACTION: If we hit threshold, change query string. Otherwise, keep searching

				$trydifferent=0;

				if($totalmatches == 0){
					#print "#### DEBUG: GOT NO MATCHES ####\n";
					if($noresults == 1){
						#print "#### DEBUG: GOT NO RESULTS ####\n";
						if($nzeromatch == 0){
							$fpzeromatch=$cur;  # Record "when" this happened
						}

						$nzeromatch++;

						# The continue = 0 condition was set above, anyway
						if($cnnoresults > $limcnnoresults){
							# If we get "no results" after 25 pages we quit
							# print "#### Bing showed no results for our query, after retries. Quitting.... ####\n";
							$continue=0;
						}
						else{
							# Before we were trying the same query. But it does not really make sense.
							$trydifferent=1;
						}
					}
					else{
						#print "#### DEBUG: GOT BLANK PAGE ####\n";

						# If we dn't even get a page that says "no results", we assume we have been banned. Back off, and retry.
						if($nzeromatch == 0){
							$fpzeromatch=$cur;  # Record "when" this happened
							#print "# Offset: $fpzeromatch\n";
						}

						$nzeromatch++;
						$czeromatch++;

						# If it is the first time the main regex fails, chances are that bing has "banned" us. Wait a bit, and retry.


							#print "#### DEBUG: SLEEP A BIT ####\n";
							sleep(rand(($nzeromatch)*2));

							if($nzeromatch > $limblankpage || $czeromatch  > $limblankpage){
								# If we get nothing after 25 pages we quit
								#print "#### Bing has banned us. Quitting.... ####\n";
								$continue=0;
							}

							if($plevel > 0 && $nzeromatch > 2){
								$trydifferent=1;
							}

					}
				}
				else{
					# We did get search results

					# If all were collisions, count the number of collisions to exit on threshold.
					if($collisions >= $matches){
						#print "#### DEBUG: GOT ALL COLLISIONS ####\n";
						$ncollisions++;
						$ccollisions++;  # Cumulative colissions even across changes in search string

						if( ($phase >= 2 && $ccollisions > $limccollisions) || ($plevel > 0 && $phase > 0 && $ccollisions > $limccollisions)){
							# If we get nothing after 100 pages (across change of query strings), and we are in 
							# phase 2 or more, we quit
							# print "#### Couldn't get any new results. Quitting.... ####\n";
							$continue=0;
						}

						if($phase == 0){
							if($subphase == 0){
								if($ncollisions > $cphaseonesubone){
									$trydifferent=1;
								}
							}
							else{
								if($ncollisions > $csubphase){
									$trydifferent=1;
								}								
							}
						}
						elsif($phase == 1){
							if($ncollisions > $cphaseonesubone){
								$trydifferent=1;
							}
						}
						elsif($phase == 2){
							if($ncollisions > $csubphase){
								$trydifferent=1;
							}
						}
						else{
							if($ncollisions > $csubphase){
								$trydifferent=1;
							}
						}

						$cur= $cur+ $totalmatches;
					}
					else{
						$difmatch=$totalmatches-$matches;
						# Not all results were collisions. So get the next search results.
						$cur= $cur+$totalmatches;
						$ncollisions=0;
						$ccollisions=0;
						#print "### GOT NEW DATA ####\n"
					}

					# If last search didn't have "Next results", we have no other option than changing the query stringg
					if($hasnext == 0){
						#print "#### DEBUG: NO NEXT RESULTS ####\n";
						$trydifferent=1;
					}

				}  # if/else on whether there were matches

				if($trydifferent == 1){
					# subfase0: no pongo nada extra
					# subfase1: tipos de archivos
					# subfase2: elimino sites
					# subfase3: elimino sites sin www;
					# subfase4: tipos de archivo + elimino sites	
					# subfase5: tipos de archivo ´+ sites sin www

					if($subphase == 0){
						$subphase=5;
						$ftindex=0;
						$ftstr="+filetype%3A$filetypesstr[$ftindex]";
						$ncollisions=0;
						$cur=1; # Since we change the search keys, we retrieve the first page of results
					}
					elsif($subphase == 1){
						if($ftindex >= $#filetypesstr){
							$ftstr="";
							$ftindex=0;
							$subphase=2;
							undef @remsites;
							$sitesstr=GetRemSitesString($pdomain, $czone, 0);
						}
						else{
							$ftindex++;
							$ftstr="+filetype%3A$filetypesstr[$ftindex]";
						}

						$ncollisions=0;
						$cur=1; # Since we change the search keys, we retrieve the first page of results
					}
					elsif($subphase == 2){
						$subphase=3;
						undef @remsites;
						$sitesstr=GetRemSitesString($pdomain, $czone, 1);
						$ncollisions=0;
						$cur=1; # Since we change the search keys, we retrieve the first page of results
					}
					elsif($subphase == 3){
						$subphase=4;
						undef @remsites;
						$sitesstr=GetRemSitesString($pdomain, $czone, 0);
						$ftindex=0;
						$ftstr="+filetype%3A$filetypesstr[$ftindex]";
						$ncollisions=0;
						$cur=1; # Since we change the search keys, we retrieve the first page of results
					}
					elsif($subphase == 4){
						if($ftindex >= $#filetypesstr){
							$subphase=5;
							$ftindex=0;
							$ftstr="+filetype%3A$filetypesstr[$ftindex]";
							undef @remsites;
							$sitesstr=GetRemSitesString($pdomain, $czone, 1);
						}
						else{
							$ftindex++;
							$ftstr="+filetype%3A$filetypesstr[$ftindex]";
						}

						$ncollisions=0;
						$cur=1; # Since we change the search keys, we retrieve the first page of results
					}
					elsif($subphase ==5){
						if($ftindex < $#filetypesstr){
							$ftindex++;
							$ftstr="+filetype%3A$filetypesstr[$ftindex]";
							$ncollisions=0;
							$cur=1; # Since we change the search keys, we retrieve the first page of results
						}
						else{
							$ftstr="";
							$ftindex=0;
							$sitesstr="";
							$subphase=0;

							if($phase == 0){
								# print "#### Phase 1  ####\n";
								$phase=1;
								$letterkey="a";
								$heur="+$letterkey";
								$ncollisions=0;
								$cur=1; # Since we change the search keys, we retrieve the first page of results
							}
							elsif($phase == 1){
								# In phase 1 we search for the domain name and one letter
								if($letterkey eq "z"){
									$letterkey="a";
									$numkey="0";
									$heur="+$numkey";
									$ncollisions=0;
									$phase=2;
									$cur=1; # Since we change the search keys, we retrieve the first page of results
									# print "#### Phase 2  ####\n";
								}
								else{
									$letterkey++;
									$ncollisions=0;
									$heur="+$letterkey";
									$cur=1; # Since we change the search keys, we retrieve the first page of results
								}
							}
							elsif($phase == 2){
								# In phase 2 we search for the domain name and a number

								if($numkey >= 10){
									$letterkey="a";
									$numkey=0;
									$heur="+$letterkey+$numkey";
									$ncollisions=0;
									$phase=3;
									$cur=1; # Since we change the search keys, we retrieve the first page of results
									# print "#### Phase 3  ####\n";
								}
								else{
									$numkey++;
									$ncollisions=0;
									$cur=1; # Since we change the search keys, we retrieve the first page of results
									$heur="+$numkey";
								}
							}
							else{
								# In phase #3, we search for the domainname, plus a letter and a number
								if($numkey > 10){
									if($letterkey eq "z"){
										# print "#### Tried my best. Quitting.... ####\n";
										$continue=0;
									}
									else{
										$letterkey++;
										$numkey=0;
										$cur=1; # Since we change the search keys, we retrieve the first page of results
										$heur="+$letterkey+$numkey";
										$ncollisions=0;
									}
								}
								else{
									$numkey++;
									$ncollisions=0;
									$cur=1; # Since we change the search keys, we retrieve the first page of results
									$heur="+$letterkey+$numkey";
								}
							} #end oh phase 
						} # end of else in subphase 5

					} #end of subphase

				}  # End of "trydifferent"
			}  # End of $? (result of wget)
			else{
#				print "# \$? was different from 0 (\$?== $?, command: '$stringcommand')\n";
				$totalqueries--;
			}
#		}while($? == 0 && $continue == 1);
		}while($continue == 1);

		if($maxresults > 0 && $#$pdomain >= $maxresults){
			return;
		}



}



sub GetDomainsFromBingByAddrQuick{
#		GetDomainsFromCrawler(\@domain, $czone, $level, $maxresults);	
		my($pdomain, $pzone, $pmaxresults, $intprotocol) = @_;
		my $czone=$pzone;

		$plevel=1;
		$MAXREMOVEDSITES=25;

# Now set from the config file
#		$MAXTIMEPERIOD=10;

#		$maxtime=$MAXTIMEPERIOD;

#		if($maxtime < 5){
#			$maxtime=5;
#		}

		$MAXCNNORESULTS=60;
		$MINCNNORESULTS= 10;

		$limcnnoresults= $MAXCNNORESULTS/($plevel+1);

		if($limcnnoresults < $MINCNNORESULTS){
			$limcnnoresults= $MINCNNORESULTS;
		}

		$MAXCCOLLISIONS=30;
		$MINCCOLISSIONS= 10;
		$limccollisions= $MAXCCOLLISIONS/($plevel+1);

		if($limccollisions < $MINCCOLISSIONS){
			$limccollisions= $MINCCOLISSIONS;
		}


		# This triggers a change in search string after cumulative number of pages that resulted
		# in 100% collisions
		$cphaseonesubone= $limccollisions/5;
		if($cphaseonesubone<2){
			$cphaseonesubone=2;
		}

		# This triggers changes in subphases that should change faster
		$csubphase= $limccollisions/5;
		if($csubphase < 2){
			$csubphase=2;
		}

		$MAXBLANKPAGE= 25;
		$MINBLANKPAGE= 5;

		$limblankpage= $MAXBLANKPAGE/($plevel+1);

		if($limblankpage < $MINBLANKPAGE){
			$limblankpage= $MINBLANKPAGE;
		}


		$maxresults= $pmaxresults;


		if($intprotocol == 4){
			$intprot="-4";
			$sengine="www.bing.com";
		}
		elsif($intprotocol == 6){
			$intprot="-6";
			$sengine="www2.bing.com";
		}
		else{
			$intprot="";
			$sengine="www.bing.com";
		}


		$cur= 1;
		$nresults=0;
		$fpzeromatch=0;
		$nzeromatch=0;
		$czeromatch=0;
		$ncollisions=0;
		$ccollisions=0;
		$cnnoresults=0;
		$continue=1;
		$phase=0;
		$subphase=0;
		$heur="";

		$letterkey="a";
		$heur="+$letterkey";


		$sitesstr="";
		$ftstr="";
		$remsites="";
		$totalqueries=0;

		do{   # Iteration for each page
			$matches=0;
			$totalmatches=0;
			$collisions=0;
			$hasnext=0;
			$noresults=0;

			# --user-agent "Mozilla/4.0"  works fine most of the time. curl || wget original headers cause trouble.
			$useragent=GetRandomUserAgent();

			if($cur == 1){
						$stringcommand="$wwwclient $intprot $wwwclient_sopt $wwwclient_tout $wwwclient_agnt\"$useragent\" \"https://$sengine/search?q=ip%3A$czone$sitesstr$ftstr$heur\"";
						@revname=`$wwwclient $intprot $wwwclient_sopt $wwwclient_tout $wwwclient_agnt"$useragent" "https://$sengine/search?q=ip%3A$czone$sitesstr$ftstr$heur"`;
			}
			else{

						$stringcommand="$wwwclient $intprot $wwwclient_sopt $wwwclient_tout $wwwclient_agnt\"$useragent\" \"https://$sengine/search?q=ip%3A$czone$sitesstr$ftstr$heur&first=$cur\"";
						@revname=`$wwwclient $intprot $wwwclient_sopt $wwwclient_tout $wwwclient_agnt"$useragent" "https://$sengine/search?q=ip%3A$czone$sitesstr$ftstr$heur&first=$cur"`;
			}
			$totalqueries++;

			if($? == 0){
				foreach $line (@revname){
					if($line=~ m/<div class="sw_next">[^<]+<\/div><\/a>/){
						$hasnext=1;
					}

					if($line=~ m/<ol id="b_results" role="main" aria-label="[^"]+"><li class="b_no">\s*<h1>/){
						$noresults=1;
						$cnnoresults++;

						if($cnnoresults >= $limcnnoresults){
							$continue=0;
						}
					}

					while($continue && $line =~ s/<h2[^>]*><a href="https?[^"\/]*\/\/([^"\/]*)//){
						$totalmatches++;
						$grabbeddomain= $1;


						$matches++;
						$nzeromatch=0;
						$czeromatch=0;

						for($i=0; $i<= $#$pdomain; $i++){
							if($pdomain->[$i] eq $grabbeddomain){
								$collisions++;
								last;
							}
						}

						if($i > $#$pdomain){
							if($maxresults < 0 || ($maxresults >= 0 &&  $#$pdomain < $maxresults)){
								$pdomain->[$#$pdomain + 1]=$grabbeddomain;
								print "$grabbeddomain\n";
								$nresults++;
							}
							else{
								$continue=0;
							}
						}

						if($maxresults >= 0 && $#$pdomain >=$maxresults){
							$continue=0;
						} 
					}
				}

				#  There are a number of posible results:
				#
				# 1) We didn't get any results:
				#     a) because search engine banned us (condition: no matches, but no "no results" page)
				#        ACTION: Wait for some time, and retry with same query string and same "results page"
				#     b) Because the query string didn't yield results (condition: "no results" page)
				#        ACTION: First couple of times, wait a bit, and retry. Otherwise, change the query string. And
				#                if we hit the threshold of "no results", quit.
				# 2) We got results:
				#     a) Some were new results
				#        ACTION: Keep searching
				#     b) All were collisions
				#        ACTION: If we hit threshold, change query string. Otherwise, keep searching

				$trydifferent=0;

				if($totalmatches == 0){
					#print "#### DEBUG: GOT NO MATCHES ####\n";
					if($noresults == 1){
						#print "#### DEBUG: GOT NO RESULTS ####\n";
						if($nzeromatch == 0){
							$fpzeromatch=$cur;  # Record "when" this happened
						}

						$nzeromatch++;

						# The continue = 0 condition was set above, anyway
						if($cnnoresults > $limcnnoresults){
							# If we get "no results" after 25 pages we quit
							# print "#### Bing showed no results for our query, after retries. Quitting.... ####\n";
							$continue=0;
						}
						else{
							# Before we were trying the same query. But it does not really make sense.
							$trydifferent=1;
						}
					}
					else{
						#print "#### DEBUG: GOT BLANK PAGE ####\n";

						# If we dn't even get a page that says "no results", we assume we have been banned. Back off, and retry.
						if($nzeromatch == 0){
							$fpzeromatch=$cur;  # Record "when" this happened
							#print "# Offset: $fpzeromatch\n";
						}

						$nzeromatch++;
						$czeromatch++;

						# If it is the first time the main regex fails, chances are that bing has "banned" us. Wait a bit, and retry.


							#print "#### DEBUG: SLEEP A BIT ####\n";
							sleep(rand(($nzeromatch)*2));

							if($nzeromatch > $limblankpage || $czeromatch  > $limblankpage){
								# If we get nothing after 25 pages we quit
								#print "#### Bing has banned us. Quitting.... ####\n";
								$continue=0;
							}

							if($plevel > 0 && $nzeromatch > 2){
								$trydifferent=1;
							}

					}
				}
				else{
					# We did get search results

					# If all were collisions, count the number of collisions to exit on threshold.
					if($collisions >= $matches){
						#print "#### DEBUG: GOT ALL COLLISIONS ####\n";
						$ncollisions++;
						$ccollisions++;  # Cumulative colissions even across changes in search string

						if( ($phase >= 2 && $ccollisions > $limccollisions) || ($plevel > 0 && $phase > 0 && $ccollisions > $limccollisions)){
							# If we get nothing after 100 pages (across change of query strings), and we are in 
							# phase 2 or more, we quit
							# print "#### Couldn't get any new results. Quitting.... ####\n";
							$continue=0;
						}

						if($phase == 0){
							if($subphase == 0){
								if($ncollisions > $cphaseonesubone){
									$trydifferent=1;
								}
							}
							else{
								if($ncollisions > $csubphase){
									$trydifferent=1;
								}								
							}
						}
						elsif($phase == 1){
							if($ncollisions > $cphaseonesubone){
								$trydifferent=1;
							}
						}
						elsif($phase == 2){
							if($ncollisions > $csubphase){
								$trydifferent=1;
							}
						}
						else{
							if($ncollisions > $csubphase){
								$trydifferent=1;
							}
						}

						$cur= $cur+ $totalmatches;
					}
					else{
						$difmatch=$totalmatches-$matches;
						# Not all results were collisions. So get the next search results.
						$cur= $cur+$totalmatches;
						$ncollisions=0;
						$ccollisions=0;
						#print "### GOT NEW DATA ####\n"
					}

					# If last search didn't have "Next results", we have no other option than changing the query stringg
					if($hasnext == 0){
						#print "#### DEBUG: NO NEXT RESULTS ####\n";
						$trydifferent=1;
					}

				}  # if/else on whether there were matches

				if($trydifferent == 1){
					# subfase0: no pongo nada extra
					# subfase1: tipos de archivos
					# subfase2: elimino sites
					# subfase3: elimino sites sin www;
					# subfase4: tipos de archivo + elimino sites	
					# subfase5: tipos de archivo ´+ sites sin www

					if($subphase == 0){
						$subphase=5;
						$ftindex=0;
						$ftstr="+filetype%3A$filetypesstr[$ftindex]";
						$ncollisions=0;
						$cur=1; # Since we change the search keys, we retrieve the first page of results
					}
					elsif($subphase == 1){
						if($ftindex >= $#filetypesstr){
							$ftstr="";
							$ftindex=0;
							$subphase=2;
							undef @remsites;
							$sitesstr=GetRemSitesString($pdomain, $czone, 0);
						}
						else{
							$ftindex++;
							$ftstr="+filetype%3A$filetypesstr[$ftindex]";
						}

						$ncollisions=0;
						$cur=1; # Since we change the search keys, we retrieve the first page of results
					}
					elsif($subphase == 2){
						$subphase=3;
						undef @remsites;
						$sitesstr=GetRemSitesString($pdomain, $czone, 1);
						$ncollisions=0;
						$cur=1; # Since we change the search keys, we retrieve the first page of results
					}
					elsif($subphase == 3){
						$subphase=4;
						undef @remsites;
						$sitesstr=GetRemSitesString($pdomain, $czone, 0);
						$ftindex=0;
						$ftstr="+filetype%3A$filetypesstr[$ftindex]";
						$ncollisions=0;
						$cur=1; # Since we change the search keys, we retrieve the first page of results
					}
					elsif($subphase == 4){
						if($ftindex >= $#filetypesstr){
							$subphase=5;
							$ftindex=0;
							$ftstr="+filetype%3A$filetypesstr[$ftindex]";
							undef @remsites;
							$sitesstr=GetRemSitesString($pdomain, $czone, 1);
						}
						else{
							$ftindex++;
							$ftstr="+filetype%3A$filetypesstr[$ftindex]";
						}

						$ncollisions=0;
						$cur=1; # Since we change the search keys, we retrieve the first page of results
					}
					elsif($subphase ==5){
						if($ftindex < $#filetypesstr){
							$ftindex++;
							$ftstr="+filetype%3A$filetypesstr[$ftindex]";
							$ncollisions=0;
							$cur=1; # Since we change the search keys, we retrieve the first page of results
						}
						else{
							$ftstr="";
							$ftindex=0;
							$sitesstr="";
							$subphase=0;

							if($phase == 0){
								# print "#### Phase 1  ####\n";
								$phase=1;
								$letterkey="a";
								$heur="+$letterkey";
								$ncollisions=0;
								$cur=1; # Since we change the search keys, we retrieve the first page of results
							}
							elsif($phase == 1){
								# In phase 1 we search for the domain name and one letter
								if($letterkey eq "z"){
									$letterkey="a";
									$numkey="0";
									$heur="+$numkey";
									$ncollisions=0;
									$phase=2;
									$cur=1; # Since we change the search keys, we retrieve the first page of results
									# print "#### Phase 2  ####\n";
								}
								else{
									$letterkey++;
									$ncollisions=0;
									$heur="+$letterkey";
									$cur=1; # Since we change the search keys, we retrieve the first page of results
								}
							}
							elsif($phase == 2){
								# In phase 2 we search for the domain name and a number

								if($numkey >= 10){
									$letterkey="a";
									$numkey=0;
									$heur="+$letterkey+$numkey";
									$ncollisions=0;
									$phase=3;
									$cur=1; # Since we change the search keys, we retrieve the first page of results
									# print "#### Phase 3  ####\n";
								}
								else{
									$numkey++;
									$ncollisions=0;
									$cur=1; # Since we change the search keys, we retrieve the first page of results
									$heur="+$numkey";
								}
							}
							else{
								# In phase #3, we search for the domainname, plus a letter and a number
								if($numkey > 10){
									if($letterkey eq "z"){
										# print "#### Tried my best. Quitting.... ####\n";
										$continue=0;
									}
									else{
										$letterkey++;
										$numkey=0;
										$cur=1; # Since we change the search keys, we retrieve the first page of results
										$heur="+$letterkey+$numkey";
										$ncollisions=0;
									}
								}
								else{
									$numkey++;
									$ncollisions=0;
									$cur=1; # Since we change the search keys, we retrieve the first page of results
									$heur="+$letterkey+$numkey";
								}
							} #end oh phase 
						} # end of else in subphase 5

					} #end of subphase

				}  # End of "trydifferent"
			}  # End of $? (result of wget)
			else{
#				print "# \$? was different from 0 (\$?== $?, command: '$stringcommand')\n";
				$totalqueries--;
			}
#		}while($? == 0 && $continue == 1);
		}while($continue == 1);

		if($maxresults > 0 && $#$pdomain >= $maxresults){
			return;
		}



}


sub GetDomainsFromBingByAddrOld{
#		GetDomainsFromCrawler(\@domain, $czone, $level, $maxresults);	
		my($pdomain, $pzone, $pmaxresults, $intprotocol) = @_;
		my $czone=$pzone;

		$MAXREMOVEDSITES=25;

# Now set from the config file
#		$MAXTIMEPERIOD=10;

#		$maxtime=$MAXTIMEPERIOD;

#		if($maxtime < 5){
#			$maxtime=5;
#		}

		$MAXCNNORESULTS=125;
		$MINCNNORESULTS= 10;

		$limcnnoresults= $MAXCNNORESULTS;

		if($limcnnoresults < $MINCNNORESULTS){
			$limcnnoresults= $MINCNNORESULTS;
		}

		$MAXCCOLLISIONS=150;
		$MINCCOLISSIONS= 10;
		$limccollisions= $MAXCCOLLISIONS;

		if($limccollisions < $MINCCOLISSIONS){
			$limccollisions= $MINCCOLISSIONS;
		}


		# This triggers a change in search string after cumulative number of pages that resulted
		# in 100% collisions
		$cphaseonesubone= $limccollisions/3;
		if($cphaseonesubone<4){
			$cphaseonesubone=4;
		}

		# This triggers changes in subphases that should change faster
		$csubphase= $limccollisions/4;
		if($csubphase < 4){
			$csubphase=4;
		}

		$MAXBLANKPAGE= 25;
		$MINBLANKPAGE= 5;

		$limblankpage= $MAXBLANKPAGE;

		if($limblankpage < $MINBLANKPAGE){
			$limblankpage= $MINBLANKPAGE;
		}


		$maxresults= $pmaxresults;


		if($intprotocol == 4){
			$intprot="-4";
			$sengine="www.bing.com";
		}
		elsif($intprotocol == 6){
			$intprot="-6";
			$sengine="www2.bing.com";
		}
		else{
			$intprot="";
			$sengine="www.bing.com";
		}


		$cur= 1;
		$nresults=0;
		$fpzeromatch=0;
		$nzeromatch=0;
		$czeromatch=0;
		$ncollisions=0;
		$ccollisions=0;
		$cnnoresults=0;
		$continue=1;
		$phase=0;
		$subphase=0;
		$heur="";
		$sitesstr="";
		$ftstr="";
		$remsites="";
		$totalqueries=0;

		do{   # Iteration for each page
			$matches=0;
			$totalmatches=0;
			$collisions=0;
			$hasnext=0;
			$noresults=0;

			# --user-agent "Mozilla/4.0"  works fine most of the time. curl || wget original headers cause trouble.
			$useragent=GetRandomUserAgent();

if($cur > 1){
			$stringcommand="$wwwclient $intprot $wwwclient_sopt $wwwclient_tout $wwwclient_agnt\"$useragent\" \"https://$sengine/search?q=ip%3A$czone$sitesstr$ftstr$heur&first=$cur\"";

#print "Pido: $stringcommand\n";
			@revname=`$wwwclient $intprot $wwwclient_sopt $wwwclient_tout $wwwclient_agnt"$useragent" "https://$sengine/search?q=ip%3A$czone$sitesstr$ftstr$heur&first=$cur"`;

}
else{
			$stringcommand="$wwwclient $intprot $wwwclient_sopt $wwwclient_tout $wwwclient_agnt\"$useragent\" \"https://$sengine/search?q=ip%3A$czone$sitesstr$ftstr$heur\"";

#print "Pido: $stringcommand\n";
			@revname=`$wwwclient $intprot $wwwclient_sopt $wwwclient_tout $wwwclient_agnt"$useragent" "https://$sengine/search?q=ip%3A$czone$sitesstr$ftstr$heur"`;

}



			$totalqueries++;

			if($? == 0){
				foreach $line (@revname){
					if($line=~ m/<div class="sw_next">[^<]+<\/div><\/a>/){
						$hasnext=1;
					}

					if($line=~ m/<ol id="b_results" role="main" aria-label="[^"]+"><li class="b_no">\s*<h1>/){
						$noresults=1;
						$cnnoresults++;

						if($cnnoresults >= $limcnnoresults){
							$continue=0;
						}
					}

					while($continue && $line =~ s/<h2[^>]*><a href="https?[^"\/]*\/\/([^"\/]*)//){
						$totalmatches++;
						$grabbeddomain= $1;

						$matches++;
						$nzeromatch=0;
						$czeromatch=0;

						for($i=0; $i<= $#$pdomain; $i++){
							if($pdomain->[$i] eq $grabbeddomain){
								$collisions++;
								last;
							}
						}

						if($i > $#$pdomain){
							if($maxresults < 0 || ($maxresults >= 0 &&  $#$pdomain < $maxresults)){
								$pdomain->[$#$pdomain + 1]=$grabbeddomain;
								print "$grabbeddomain\n";
								$nresults++;
							}
							else{
								$continue=0;
							}
						}

						if($maxresults >= 0 && $#$pdomain >=$maxresults){
							$continue=0;
						} 
					}
				}

				#  There are a number of posible results:
				#
				# 1) We didn't get any results:
				#     a) because search engine banned us (condition: no matches, but no "no results" page)
				#        ACTION: Wait for some time, and retry with same query string and same "results page"
				#     b) Because the query string didn't yield results (condition: "no results" page)
				#        ACTION: First couple of times, wait a bit, and retry. Otherwise, change the query string. And
				#                if we hit the threshold of "no results", quit.
				# 2) We got results:
				#     a) Some were new results
				#        ACTION: Keep searching
				#     b) All were collisions
				#        ACTION: If we hit threshold, change query string. Otherwise, keep searching

				if($totalmatches == 0){
					#print "#### DEBUG: GOT NO MATCHES ####\n";
					if($noresults == 1){
						#print "#### DEBUG: GOT NO RESULTS ####\n";
						if($nzeromatch == 0){
							$fpzeromatch=$cur;  # Record "when" this happened
						}

						$nzeromatch++;

						# The continue = 0 condition was set above, anyway
						if($cnnoresults > $limcnnoresults){
							# If we get "no results" after 25 pages we quit
							# print "#### Bing showed no results for our query, after retries. Quitting.... ####\n";
							$continue=0;
						}
					}
					else{
						#print "#### DEBUG: GOT BLANK PAGE ####\n";

						# If we dn't even get a page that says "no results", we assume we have been banned. Back off, and retry.
						if($nzeromatch == 0){
							$fpzeromatch=$cur;  # Record "when" this happened
							#print "# Offset: $fpzeromatch\n";
						}

						$nzeromatch++;
						$czeromatch++;

						# If it is the first time the main regex fails, chances are that bing has "banned" us. Wait a bit, and retry.
						#print "#### DEBUG: SLEEP A BIT ####\n";
						sleep(rand(($nzeromatch)*15));

						if($nzeromatch > $limblankpage || $czeromatch  > $limblankpage){
							# If we get nothing after 25 pages we quit
							#print "#### Bing has banned us. Quitting.... ####\n";
							$continue=0;
						}

						if($nzeromatch > 2){
							$continue=0;
						}
					}
				}
				else{
					# We did get search results

					# If all were collisions, count the number of collisions to exit on threshold.
					if($collisions >= $matches){
						#print "#### DEBUG: GOT ALL COLLISIONS ####\n";
						$ncollisions++;
						$ccollisions++;  # Cumulative colissions even across changes in search string

						if($ccollisions > $limccollisions){
							# If we get nothing after 100 pages (across change of query strings), and we are in 
							# phase 2 or more, we quit
							# print "#### Couldn't get any new results. Quitting.... ####\n";
							$continue=0;
						}

						$cur= $cur+ $totalmatches;
					}
					else{
						$difmatch=$totalmatches-$matches;
						# Not all results were collisions. So get the next search results.
						$cur= $cur+$totalmatches;
						$ncollisions=0;
						$ccollisions=0;
						#print "### GOT NEW DATA ####\n"
					}

					# If last search didn't have "Next results", we have no other option than changing the query stringg
					if($hasnext == 0){
						#print "#### DEBUG: NO NEXT RESULTS ####\n";
						$continue=0;
					}

				}  # if/else on whether there were matches

			}  # End of $? (result of wget)
			else{
#				print "# \$? was different from 0 (\$?== $?, command: '$stringcommand')\n";
				$totalqueries--;
			}
#		}while($? == 0 && $continue == 1);
		}while($continue == 1);

}



# XXX Check (old unused function?)

sub GetDomainsFromBingBis{ # bis
#		GetDomainsFromCrawler(\@domain, $czone, $level, $maxresults);	
		my($pdomain, $pzone, $plevel, $pmaxresults) = @_;
		my $czone=$pzone;

		$MAXREMOVEDSITES=25;

# Now set from the config file
#		$MAXTIMEPERIOD=10;

#		$maxtime=$MAXTIMEPERIOD;

#		if($maxtime < 5){
#			$maxtime=5;
#		}

		$MAXCNNORESULTS= 25; #50
		$MINCNNORESULTS= 5;

		$limcnnoresults= $MAXCNNORESULTS/($plevel+1);

		if($limcnnoresults < $MINCNNORESULTS){
			$limcnnoresults= $MINCNNORESULTS;
		}

		$MAXCCOLLISIONS= 25; # 50
		$MINCCOLISSIONS= 5;
		$limccollisions= $MAXCCOLLISIONS/($plevel+1);

		if($limccollisions < $MINCCOLISSIONS){
			$limccollisions= $MINCCOLISSIONS;
		}


		# This triggers a change in search string after cumulative number of pages that resulted
		# in 100% collisions
		$cphaseonesubone= $limccollisions/3;
		if($cphaseonesubone<4){
			$cphaseonesubone=4;
		}

		# This triggers changes in subphases that should change faster
		$csubphase= $limccollisions/4;
		if($csubphase < 4){
			$csubphase=4;
		}

		$MAXBLANKPAGE= 10;
		$MINBLANKPAGE= 5;

		$limblankpage= $MAXBLANKPAGE/($plevel+1);

		if($limblankpage < $MINBLANKPAGE){
			$limblankpage= $MINBLANKPAGE;
		}


		$maxresults= $pmaxresults;

		$cur= 1;
		$nresults=0;
		$fpzeromatch=0;
		$nzeromatch=0;
		$czeromatch=0;
		$ncollisions=0;
		$ccollisions=0;
		$cnnoresults=0;
		$continue=1;
		$phase=0;
		$subphase=0;
		$heur="";
		$sitesstr="";
		$ftstr="";
		$remsites="";
		$totalqueries=0;

		do{   # Iteration for each page
			$matches=0;
			$totalmatches=0;
			$collisions=0;
			$hasnext=0;
			$noresults=0;

			# --user-agent "Mozilla/4.0"  works fine most of the time. curl || wget original headers cause trouble.
			$useragent=GetRandomUserAgent();

			@revname=`$wwwclient $wwwclient_sopt $wwwclient_tout $wwwclient_agnt"$useragent" "https://www.bing.com/search?q=site%3A$czone$sitesstr$ftstr$heur&first=$cur"`;

			$totalqueries++;

			if($? == 0){
				foreach $line (@revname){
					if($line=~ m/<div class="sw_next">[^<]+<\/div><\/a>/){
						$hasnext=1;
					}

					if($line=~ m/<ol id="b_results" role="main" aria-label="[^"]+"><li class="b_no">\s*<h1>/){
						$noresults=1;
						$cnnoresults++;

						if($cnnoresults >= $limcnnoresults){
							$continue=0;
						}
					}

					while($continue && $line =~ s/<h2[^>]*><a href="https?[^"\/]*\/\/([^"\/]*)//){
						$totalmatches++;
						$grabbeddomain= $1;


						if($grabbeddomain !~ m/$czone(:\d+)?/){
							#print "##### DEBUG: Discarded $grabbeddomain\n";
							next;
						}

						$matches++;
						$nzeromatch=0;
						$czeromatch=0;

						for($i=0; $i<= $#$pdomain; $i++){
							if($pdomain->[$i] eq $grabbeddomain){
								$collisions++;
								last;
							}
						}

						if($i > $#$pdomain){
							if($maxresults < 0 || ($maxresults >= 0 &&  $#$pdomain < $maxresults)){
								$pdomain->[$#$pdomain + 1]=$grabbeddomain;
								print "$grabbeddomain\n";
								$nresults++;
							}
							else{
								$continue=0;
							}
						}

						if($maxresults >= 0 && $#$pdomain >=$maxresults){
							$continue=0;
						} 
					}
				}

				#  There are a number of posible results:
				#
				# 1) We didn't get any results:
				#     a) because search engine banned us (condition: no matches, but no "no results" page)
				#        ACTION: Wait for some time, and retry with same query string and same "results page"
				#     b) Because the query string didn't yield results (condition: "no results" page)
				#        ACTION: First couple of times, wait a bit, and retry. Otherwise, change the query string. And
				#                if we hit the threshold of "no results", quit.
				# 2) We got results:
				#     a) Some were new results
				#        ACTION: Keep searching
				#     b) All were collisions
				#        ACTION: If we hit threshold, change query string. Otherwise, keep searching

				$trydifferent=0;

				if($totalmatches == 0){
					#print "#### DEBUG: GOT NO MATCHES ####\n";
					if($noresults == 1){
						#print "#### DEBUG: GOT NO RESULTS ####\n";
						if($nzeromatch == 0){
							$fpzeromatch=$cur;  # Record "when" this happened
						}

						$nzeromatch++;

						# The continue = 0 condition was set above, anyway
						if($cnnoresults > $limcnnoresults){
							# If we get "no results" after 25 pages we quit
							# print "#### Bing showed no results for our query, after retries. Quitting.... ####\n";
							$continue=0;
						}
						else{
							# Before we were trying the same query. But it does not really make sense.
							$trydifferent=1;
						}
					}
					else{
						#print "#### DEBUG: GOT BLANK PAGE ####\n";

						# If we dn't even get a page that says "no results", we assume we have been banned. Back off, and retry.
						if($nzeromatch == 0){
							$fpzeromatch=$cur;  # Record "when" this happened
							#print "# Offset: $fpzeromatch\n";
						}

						$nzeromatch++;
						$czeromatch++;

						# If it is the first time the main regex fails, chances are that bing has "banned" us. Wait a bit, and retry.
						#print "#### DEBUG: SLEEP A BIT ####\n";
						sleep(rand(($nzeromatch)*15));

						if($nzeromatch > $limblankpage || $czeromatch  > $limblankpage){
							# If we get nothing after 25 pages we quit
							#print "#### Bing has banned us. Quitting.... ####\n";
							$continue=0;
						}

						if($plevel > 0 && $nzeromatch > 2){
							$trydifferent=1;
						}
					}
				}
				else{
					# We did get search results

					# If all were collisions, count the number of collisions to exit on threshold.
					if($collisions >= $matches){
						#print "#### DEBUG: GOT ALL COLLISIONS ####\n";
						$ncollisions++;
						$ccollisions++;  # Cumulative colissions even across changes in search string

						if( ($phase >= 2 && $ccollisions > $limccollisions) || ($plevel > 0 && $phase > 0 && $ccollisions > $limccollisions)){
							# If we get nothing after 100 pages (across change of query strings), and we are in 
							# phase 2 or more, we quit
							# print "#### Couldn't get any new results. Quitting.... ####\n";
							$continue=0;
						}

						if($phase == 0){
							if($subphase == 0){
								if($ncollisions > $cphaseonesubone){
									$trydifferent=1;
								}
							}
							else{
								if($ncollisions > $csubphase){
									$trydifferent=1;
								}								
							}
						}
						elsif($phase == 1){
							if($ncollisions > $cphaseonesubone){
								$trydifferent=1;
							}
						}
						elsif($phase == 2){
							if($ncollisions > $csubphase){
								$trydifferent=1;
							}
						}
						else{
							if($ncollisions > $csubphase){
								$trydifferent=1;
							}
						}

						$cur= $cur+ $totalmatches;
					}
					else{
						$difmatch=$totalmatches-$matches;
						# Not all results were collisions. So get the next search results.
						$cur= $cur+$totalmatches;
						$ncollisions=0;
						$ccollisions=0;
						#print "### GOT NEW DATA ####\n"
					}

					# If last search didn't have "Next results", we have no other option than changing the query stringg
					if($hasnext == 0){
						#print "#### DEBUG: NO NEXT RESULTS ####\n";
						$trydifferent=1;
					}

				}  # if/else on whether there were matches

				if($trydifferent == 1){
					# subfase0: no pongo nada extra
					# subfase1: tipos de archivos
					# subfase2: elimino sites
					# subfase3: elimino sites sin www;
					# subfase4: tipos de archivo + elimino sites	
					# subfase5: tipos de archivo ´+ sites sin www

					if($subphase == 0){
						$subphase=1;
						$ftindex=0;
						$ftstr="+filetype%3A$filetypesstr[$ftindex]";
						$ncollisions=0;
						$cur=1; # Since we change the search keys, we retrieve the first page of results
					}
					elsif($subphase == 1){
						if($ftindex >= $#filetypesstr){
							$ftstr="";
							$ftindex=0;
							$subphase=2;
							undef @remsites;
							$sitesstr=GetRemSitesString($pdomain, $czone, 0);
						}
						else{
							$ftindex++;
							$ftstr="+filetype%3A$filetypesstr[$ftindex]";
						}

						$ncollisions=0;
						$cur=1; # Since we change the search keys, we retrieve the first page of results
					}
					elsif($subphase == 2){
						$subphase=3;
						undef @remsites;
						$sitesstr=GetRemSitesString($pdomain, $czone, 1);
						$ncollisions=0;
						$cur=1; # Since we change the search keys, we retrieve the first page of results
					}
					elsif($subphase == 3){
						$subphase=4;
						undef @remsites;
						$sitesstr=GetRemSitesString($pdomain, $czone, 0);
						$ftindex=0;
						$ftstr="+filetype%3A$filetypesstr[$ftindex]";
						$ncollisions=0;
						$cur=1; # Since we change the search keys, we retrieve the first page of results
					}
					elsif($subphase == 4){
						if($ftindex >= $#filetypesstr){
							$subphase=5;
							$ftindex=0;
							$ftstr="+filetype%3A$filetypesstr[$ftindex]";
							undef @remsites;
							$sitesstr=GetRemSitesString($pdomain, $czone, 1);
						}
						else{
							$ftindex++;
							$ftstr="+filetype%3A$filetypesstr[$ftindex]";
						}

						$ncollisions=0;
						$cur=1; # Since we change the search keys, we retrieve the first page of results
					}
					elsif($subphase ==5){
						if($ftindex < $#filetypesstr){
							$ftindex++;
							$ftstr="+filetype%3A$filetypesstr[$ftindex]";
							$ncollisions=0;
							$cur=1; # Since we change the search keys, we retrieve the first page of results
						}
						else{
							$ftstr="";
							$ftindex=0;
							$sitesstr="";
							$subphase=0;

							if($phase == 0){
								# print "#### Phase 1  ####\n";
								$phase=1;
								$letterkey="a";
								$heur="+$letterkey";
								$ncollisions=0;
								$cur=1; # Since we change the search keys, we retrieve the first page of results
							}
							elsif($phase == 1){
								# In phase 1 we search for the domain name and one letter
								if($letterkey eq "z"){
									$letterkey="a";
									$numkey="0";
									$heur="+$numkey";
									$ncollisions=0;
									$phase=2;
									$cur=1; # Since we change the search keys, we retrieve the first page of results
									# print "#### Phase 2  ####\n";
								}
								else{
									$letterkey++;
									$ncollisions=0;
									$heur="+$letterkey";
									$cur=1; # Since we change the search keys, we retrieve the first page of results
								}
							}
							elsif($phase == 2){
								# In phase 2 we search for the domain name and a number

								if($numkey >= 10001){
									$letterkey="a";
									$numkey=0;
									$heur="+$letterkey+$numkey";
									$ncollisions=0;
									$phase=3;
									$cur=1; # Since we change the search keys, we retrieve the first page of results
									# print "#### Phase 3  ####\n";
								}
								else{
									$numkey++;
									$ncollisions=0;
									$cur=1; # Since we change the search keys, we retrieve the first page of results
									$heur="+$numkey";
								}
							}
							else{
								# In phase #3, we search for the domainname, plus a letter and a number
								if($numkey > 10000){
									if($letterkey eq "z"){
										# print "#### Tried my best. Quitting.... ####\n";
										$continue=0;
									}
									else{
										$letterkey++;
										$numkey=0;
										$cur=1; # Since we change the search keys, we retrieve the first page of results
										$heur="+$letterkey+$numkey";
										$ncollisions=0;
									}
								}
								else{
									$numkey++;
									$ncollisions=0;
									$cur=1; # Since we change the search keys, we retrieve the first page of results
									$heur="+$letterkey+$numkey";
								}
							} #end oh phase 
						} # end of else in subphase 5

					} #end of subphase

				}  # End of "trydifferent"
			}  # End of $? (result of wget)

			if($? != 0){
#				print "# \$? was different from 0 (\$?== $?)\n";
				$totalqueries--;
			}
#		}while($? == 0 && $continue == 1);
		}while($continue == 1);

		if($maxresults > 0 && $#$pdomain >= $maxresults){
			return;
		}


		# We will now exclude next level domains

		undef @remsites;

		GenDomListLevel($pdomain, \@remsites, $czone, 1, $MAXREMOVEDSITES);

		$continue=1;

		for($jjj= 0; $jjj <= $#remsites && $continue == 1; $jjj++){
			$cur= 1;

			$sitesstr= GenRemListStr(\@remsites, $jjj);
			$trydifferent=0;

			do{   # Iteration for each page
				$matches=0;
				$totalmatches=0;
				$collisions=0;
				$hasnext=0;
				$noresults=0;

				$useragent=GetRandomUserAgent();

				@revname=`$wwwclient $wwwclient_sopt $wwwclient_tout $wwwclient_agnt"$useragent" "https://www.bing.com/search?q=site%3A$czone$sitesstr&first=$cur"`;

				$totalqueries++;

				if($? == 0){
					foreach $line (@revname){
						if($continue == 0){
							last;
						}

						if($line=~ m/<div class="sw_next">[^<]+<\/div><\/a>/){
							$hasnext=1;
						}

						if($line=~ m/<ol id="b_results" role="main" aria-label="[^"]+"><li class="b_no">\s*<h1>/){
							$noresults=1;
						}

						while($continue && $line =~ s/<h2[^>]*><a href="https?[^"\/]*\/\/([^"\/]*)//){
							$totalmatches++;
							$grabbeddomain= $1;

							if($grabbeddomain !~ m/$czone(:\d+)?/){
								next;
							}

							$matches++;
							$nzeromatch=0;

							for($k=0; $k<= $#$pdomain; $k++){
								if($pdomain->[$k] eq $grabbeddomain){
									$collisions++;
									last;
								}
							}


							if($k > $#$pdomain){
								if($maxresults < 0 || ($maxresults >= 0 &&  $#$pdomain < $maxresults)){
									$pdomain->[$#$pdomain + 1]=$grabbeddomain;
									print "$grabbeddomain\n";
									$nresults++;
								}
								else{
									$continue=0;
								}
							}

							if($maxresults >= 0 && $#$pdomain>=$maxresults){
								$continue=0;
							} 
						} # while() that reads lines
					}  # foreach $line

					$trydifferent=0;

					if($totalmatches == 0){
						if($noresults == 1){
							$trydifferent=1;
						}
						else{
							#print "#### DEBUG: GOT BLANK PAGE ####\n";
							$nzeromatch++;

							# If it is the first time the main regex fails, chances are that bing has "banned" us. Wait a bit, and retry.
							#print "#### DEBUG: SLEEP A BIT ####\n";
							sleep(rand(($nzeromatch)*20));

							if($nzeromatch > 5){
								#	print "#### Bing has banned us. Quitting.... ####\n";
								$trydifferent=1;
							}
						}
					}
					else{
						# We did get search results

						# If all were collisions, count the number of collisions to exit on threshold.
						if($collisions >= $matches){
							#print "#### DEBUG: GOT ALL COLLISIONS ####\n";
							$ncollisions++;

							if($ncollisions >= 10){
								$trydifferent=1;
							} 

							$cur= $cur+ $totalmatches;
						}
						else{
							$difmatch=$totalmatches-$matches;
							# Not all results were collisions. So get the next search results.
							$cur= $cur+$totalmatches;
							$ncollisions=0;
						}

						# If last search didn't have "Next results", we have no other option than changing the query stringg
						if($hasnext == 0){
							#print "#### DEBUG: NO NEXT RESULTS ####\n";
							$trydifferent=1;
						}
					}  # if/else on whether there were matches
				}  # End of $? (result of wget)

				if($? != 0){
#					print "# \$? was different from 0 (\$?== $?)\n";
					$totalqueries--;
				}
			}while($trydifferent == 0);

			if($#remsites < ($MAXREMOVEDSITES-1)){
				# If list is not complete, attempt to complete it
				GenDomListLevel($pdomain, \@remsites, $czone, 1, $MAXREMOVEDSITES);
			}
		} # for that adds new removed sites;
}




sub GetDomainsFromBingTT{
#		GetDomainsFromCrawler(\@domain, $czone, $level, $maxresults);	
		my($pdomain, $pzone, $plevel, $pmaxresults) = @_;
		my $czone=$pzone;

		$tstamp2=time;
		$MAXREMOVEDSITES=25;

# Now set from the config file
#		$MAXTIMEPERIOD=10;

#		$maxtime=$MAXTIMEPERIOD;

#		if($maxtime < 5){
#			$maxtime=5;
#		}

		$MAXCNNORESULTS=70;
		$MINCNNORESULTS= 10;

		$limcnnoresults= $MAXCNNORESULTS/($plevel+1);

		if($limcnnoresults < $MINCNNORESULTS){
			$limcnnoresults= $MINCNNORESULTS;
		}

		$MAXCCOLLISIONS= 70;
		$MINCCOLISSIONS= 10;
		$limccollisions= $MAXCCOLLISIONS/($plevel+1);

		if($limccollisions < $MINCCOLISSIONS){
			$limccollisions= $MINCCOLISSIONS;
		}


		# This triggers a change in search string after cumulative number of pages that resulted
		# in 100% collisions
		$cphaseonesubone= $limccollisions/3;
		if($cphaseonesubone<4){
			$cphaseonesubone=4;
		}

		# This triggers changes in subphases that should change faster
		$csubphase= $limccollisions/4;
		if($csubphase < 4){
			$csubphase=4;
		}

		$MAXBLANKPAGE= 25;
		$MINBLANKPAGE= 5;

		$limblankpage= $MAXBLANKPAGE/($plevel+1);

		if($limblankpage < $MINBLANKPAGE){
			$limblankpage= $MINBLANKPAGE;
		}


		$maxresults= $pmaxresults;

		$cur= 1;
		$nresults=0;
		$fpzeromatch=0;
		$nzeromatch=0;
		$czeromatch=0;
		$ncollisions=0;
		$ccollisions=0;
		$cnnoresults=0;
		$continue=1;
		$phase=0;
		$subphase=0;
		$heur="";
		$sitesstr="";
		$ftstr="";
		$remsites="";
		$totalqueries=0;

		do{   # Iteration for each page
			$matches=0;
			$totalmatches=0;
			$collisions=0;
			$hasnext=0;
			$noresults=0;

			# --user-agent "Mozilla/4.0"  works fine most of the time. curl || wget original headers cause trouble.
			$useragent=GetRandomUserAgent();

			@revname=`$wwwclient $wwwclient_sopt $wwwclient_tout $wwwclient_agnt"$useragent" "https://www.bing.com/search?q=site%3A$czone$sitesstr$ftstr$heur&first=$cur"`;

			$totalqueries++;

			if($? == 0){
				foreach $line (@revname){
				if((time - $tstamp2) > 25){
					return;
				}
					if($line=~ m/<div class="sw_next">[^<]+<\/div><\/a>/){
						$hasnext=1;
					}

					if($line=~ m/<ol id="b_results" role="main" aria-label="[^"]+"><li class="b_no">\s*<h1>/){
						$noresults=1;
						$cnnoresults++;

						if($cnnoresults >= $limcnnoresults){
							$continue=0;
						}
					}

					while($continue && $line =~ s/<h2[^>]*><a href="https?[^"\/]*\/\/([^"\/]*)//){
						$totalmatches++;
						$grabbeddomain= $1;


						if($grabbeddomain !~ m/$czone(:\d+)?/){
							#print "##### DEBUG: Discarded $grabbeddomain\n";
							next;
						}

						$matches++;
						$nzeromatch=0;
						$czeromatch=0;

						for($i=0; $i<= $#$pdomain; $i++){
							if($pdomain->[$i] eq $grabbeddomain){
								$collisions++;
								last;
							}
						}

						if($i > $#$pdomain){
							if($maxresults < 0 || ($maxresults >= 0 &&  $#$pdomain < $maxresults)){
								$pdomain->[$#$pdomain + 1]=$grabbeddomain;
								print "$grabbeddomain\n";
								$nresults++;
							}
							else{
								$continue=0;
							}
						}

						if($maxresults >= 0 && $#$pdomain >=$maxresults){
							$continue=0;
						} 
					}
				}

				#  There are a number of posible results:
				#
				# 1) We didn't get any results:
				#     a) because search engine banned us (condition: no matches, but no "no results" page)
				#        ACTION: Wait for some time, and retry with same query string and same "results page"
				#     b) Because the query string didn't yield results (condition: "no results" page)
				#        ACTION: First couple of times, wait a bit, and retry. Otherwise, change the query string. And
				#                if we hit the threshold of "no results", quit.
				# 2) We got results:
				#     a) Some were new results
				#        ACTION: Keep searching
				#     b) All were collisions
				#        ACTION: If we hit threshold, change query string. Otherwise, keep searching

				$trydifferent=0;

				if($totalmatches == 0){
					#print "#### DEBUG: GOT NO MATCHES ####\n";
					if($noresults == 1){
						#print "#### DEBUG: GOT NO RESULTS ####\n";
						if($nzeromatch == 0){
							$fpzeromatch=$cur;  # Record "when" this happened
						}

						$nzeromatch++;

						# The continue = 0 condition was set above, anyway
						if($cnnoresults > $limcnnoresults){
							# If we get "no results" after 25 pages we quit
							# print "#### Bing showed no results for our query, after retries. Quitting.... ####\n";
							$continue=0;
						}
						else{
							# Before we were trying the same query. But it does not really make sense.
							$trydifferent=1;
						}
					}
					else{
						#print "#### DEBUG: GOT BLANK PAGE ####\n";

						# If we dn't even get a page that says "no results", we assume we have been banned. Back off, and retry.
						if($nzeromatch == 0){
							$fpzeromatch=$cur;  # Record "when" this happened
							#print "# Offset: $fpzeromatch\n";
						}

						$nzeromatch++;
						$czeromatch++;

						# If it is the first time the main regex fails, chances are that bing has "banned" us. Wait a bit, and retry.
						#print "#### DEBUG: SLEEP A BIT ####\n";
						sleep(rand(($nzeromatch)*15));

						if($nzeromatch > $limblankpage || $czeromatch  > $limblankpage){
							# If we get nothing after 25 pages we quit
							#print "#### Bing has banned us. Quitting.... ####\n";
							$continue=0;
						}

						if($plevel > 0 && $nzeromatch > 2){
							$trydifferent=1;
						}
					}
				}
				else{
					# We did get search results

					# If all were collisions, count the number of collisions to exit on threshold.
					if($collisions >= $matches){
						#print "#### DEBUG: GOT ALL COLLISIONS ####\n";
						$ncollisions++;
						$ccollisions++;  # Cumulative colissions even across changes in search string

						if( ($phase >= 2 && $ccollisions > $limccollisions) || ($plevel > 0 && $phase > 0 && $ccollisions > $limccollisions)){
							# If we get nothing after 100 pages (across change of query strings), and we are in 
							# phase 2 or more, we quit
							# print "#### Couldn't get any new results. Quitting.... ####\n";
							$continue=0;
						}

						if($phase == 0){
							if($subphase == 0){
								if($ncollisions > $cphaseonesubone){
									$trydifferent=1;
								}
							}
							else{
								if($ncollisions > $csubphase){
									$trydifferent=1;
								}								
							}
						}
						elsif($phase == 1){
							if($ncollisions > $cphaseonesubone){
								$trydifferent=1;
							}
						}
						elsif($phase == 2){
							if($ncollisions > $csubphase){
								$trydifferent=1;
							}
						}
						else{
							if($ncollisions > $csubphase){
								$trydifferent=1;
							}
						}

						$cur= $cur+ $totalmatches;
					}
					else{
						$difmatch=$totalmatches-$matches;
						# Not all results were collisions. So get the next search results.
						$cur= $cur+$totalmatches;
						$ncollisions=0;
						$ccollisions=0;
						#print "### GOT NEW DATA ####\n"
					}

					# If last search didn't have "Next results", we have no other option than changing the query stringg
					if($hasnext == 0){
						#print "#### DEBUG: NO NEXT RESULTS ####\n";
						$trydifferent=1;
					}

				}  # if/else on whether there were matches

				if($trydifferent == 1){
					# subfase0: no pongo nada extra
					# subfase1: tipos de archivos
					# subfase2: elimino sites
					# subfase3: elimino sites sin www;
					# subfase4: tipos de archivo + elimino sites	
					# subfase5: tipos de archivo ´+ sites sin www

					if($subphase == 0){
						$subphase=1;
						$ftindex=0;
						$ftstr="+filetype%3A$filetypesstr[$ftindex]";
						$ncollisions=0;
						$cur=1; # Since we change the search keys, we retrieve the first page of results
					}
					elsif($subphase == 1){
						if($ftindex >= $#filetypesstr){
							$ftstr="";
							$ftindex=0;
							$subphase=2;
							undef @remsites;
							$sitesstr=GetRemSitesString($pdomain, $czone, 0);
						}
						else{
							$ftindex++;
							$ftstr="+filetype%3A$filetypesstr[$ftindex]";
						}

						$ncollisions=0;
						$cur=1; # Since we change the search keys, we retrieve the first page of results
					}
					elsif($subphase == 2){
						$subphase=3;
						undef @remsites;
						$sitesstr=GetRemSitesString($pdomain, $czone, 1);
						$ncollisions=0;
						$cur=1; # Since we change the search keys, we retrieve the first page of results
					}
					elsif($subphase == 3){
						$subphase=4;
						undef @remsites;
						$sitesstr=GetRemSitesString($pdomain, $czone, 0);
						$ftindex=0;
						$ftstr="+filetype%3A$filetypesstr[$ftindex]";
						$ncollisions=0;
						$cur=1; # Since we change the search keys, we retrieve the first page of results
					}
					elsif($subphase == 4){
						if($ftindex >= $#filetypesstr){
							$subphase=5;
							$ftindex=0;
							$ftstr="+filetype%3A$filetypesstr[$ftindex]";
							undef @remsites;
							$sitesstr=GetRemSitesString($pdomain, $czone, 1);
						}
						else{
							$ftindex++;
							$ftstr="+filetype%3A$filetypesstr[$ftindex]";
						}

						$ncollisions=0;
						$cur=1; # Since we change the search keys, we retrieve the first page of results
					}
					elsif($subphase ==5){
						if($ftindex < $#filetypesstr){
							$ftindex++;
							$ftstr="+filetype%3A$filetypesstr[$ftindex]";
							$ncollisions=0;
							$cur=1; # Since we change the search keys, we retrieve the first page of results
						}
						else{
							$ftstr="";
							$ftindex=0;
							$sitesstr="";
							$subphase=0;

							if($phase == 0){
								# print "#### Phase 1  ####\n";
								$phase=1;
								$letterkey="a";
								$heur="+$letterkey";
								$ncollisions=0;
								$cur=1; # Since we change the search keys, we retrieve the first page of results
							}
							elsif($phase == 1){
								# In phase 1 we search for the domain name and one letter
								if($letterkey eq "z"){
									$letterkey="a";
									$numkey="0";
									$heur="+$numkey";
									$ncollisions=0;
									$phase=2;
									$cur=1; # Since we change the search keys, we retrieve the first page of results
									# print "#### Phase 2  ####\n";
								}
								else{
									$letterkey++;
									$ncollisions=0;
									$heur="+$letterkey";
									$cur=1; # Since we change the search keys, we retrieve the first page of results
								}
							}
							elsif($phase == 2){
								# In phase 2 we search for the domain name and a number

								if($numkey >= 10001){
									$letterkey="a";
									$numkey=0;
									$heur="+$letterkey+$numkey";
									$ncollisions=0;
									$phase=3;
									$cur=1; # Since we change the search keys, we retrieve the first page of results
									# print "#### Phase 3  ####\n";
								}
								else{
									$numkey++;
									$ncollisions=0;
									$cur=1; # Since we change the search keys, we retrieve the first page of results
									$heur="+$numkey";
								}
							}
							else{
								# In phase #3, we search for the domainname, plus a letter and a number
								if($numkey > 10000){
									if($letterkey eq "z"){
										# print "#### Tried my best. Quitting.... ####\n";
										$continue=0;
									}
									else{
										$letterkey++;
										$numkey=0;
										$cur=1; # Since we change the search keys, we retrieve the first page of results
										$heur="+$letterkey+$numkey";
										$ncollisions=0;
									}
								}
								else{
									$numkey++;
									$ncollisions=0;
									$cur=1; # Since we change the search keys, we retrieve the first page of results
									$heur="+$letterkey+$numkey";
								}
							} #end oh phase 
						} # end of else in subphase 5

					} #end of subphase

				}  # End of "trydifferent"

			}  # End of $? (result of wget)

			if($? != 0){
#				print "# \$? was different from 0 (\$?== $?)\n";
				$totalqueries--;
			}
		}while($continue == 1);


		if($maxresults > 0 && $#$pdomain >= $maxresults){
			return;
		}


		# We will now exclude next level domains

		undef @remsites;

		GenDomListLevel($pdomain, \@remsites, $czone, 1, $MAXREMOVEDSITES);

		$continue=1;
#print "pase por seg timer\n";
		$tstamp2=time;

		for($jjj= 0; $jjj <= $#remsites && $continue == 1; $jjj++){
			$cur= 1;

			$sitesstr= GenRemListStr(\@remsites, $jjj);
			$trydifferent=0;

			do{   # Iteration for each page


				$matches=0;
				$totalmatches=0;
				$collisions=0;
				$hasnext=0;
				$noresults=0;

				$useragent=GetRandomUserAgent();

				@revname=`$wwwclient $wwwclient_sopt $wwwclient_tout $wwwclient_agnt"$useragent" "https://www.bing.com/search?q=site%3A$czone$sitesstr&first=$cur"`;

				$totalqueries++;

				if($? == 0){
					foreach $line (@revname){
						if($continue == 0){
							last;
						}

						if($line=~ m/<div class="sw_next">[^<]+<\/div><\/a>/){
							$hasnext=1;
						}

						if($line=~ m/<ol id="b_results" role="main" aria-label="[^"]+"><li class="b_no">\s*<h1>/){
							$noresults=1;
						}

						while($continue && $line =~ s/<h2[^>]*><a href="https?[^"\/]*\/\/([^"\/]*)//){
							$totalmatches++;
							$grabbeddomain= $1;

							if($grabbeddomain !~ m/$czone(:\d+)?/){
								next;
							}

							$matches++;
							$nzeromatch=0;

							for($k=0; $k<= $#$pdomain; $k++){
								if($pdomain->[$k] eq $grabbeddomain){
									$collisions++;
									last;
								}
							}


							if($k > $#$pdomain){
								if($maxresults < 0 || ($maxresults >= 0 &&  $#$pdomain < $maxresults)){
									$pdomain->[$#$pdomain + 1]=$grabbeddomain;
									print "$grabbeddomain\n";
									$nresults++;
								}
								else{
									$continue=0;
								}
							}

							if($maxresults >= 0 && $#$pdomain>=$maxresults){
								$continue=0;
							} 
						} # while() that reads lines
					}  # foreach $line

					$trydifferent=0;

					if($totalmatches == 0){
						if($noresults == 1){
							$trydifferent=1;
						}
						else{
							#print "#### DEBUG: GOT BLANK PAGE ####\n";
							$nzeromatch++;

							# If it is the first time the main regex fails, chances are that bing has "banned" us. Wait a bit, and retry.
							#print "#### DEBUG: SLEEP A BIT ####\n";
							sleep(rand(($nzeromatch)*20));

							if($nzeromatch > 5){
								#	print "#### Bing has banned us. Quitting.... ####\n";
								$trydifferent=1;
							}
						}
					}
					else{
						# We did get search results

						# If all were collisions, count the number of collisions to exit on threshold.
						if($collisions >= $matches){
							#print "#### DEBUG: GOT ALL COLLISIONS ####\n";
							$ncollisions++;

							if($ncollisions >= 10){
								$trydifferent=1;
							} 

							$cur= $cur+ $totalmatches;
						}
						else{
							$difmatch=$totalmatches-$matches;
							# Not all results were collisions. So get the next search results.
							$cur= $cur+$totalmatches;
							$ncollisions=0;
						}

						# If last search didn't have "Next results", we have no other option than changing the query stringg
						if($hasnext == 0){
							#print "#### DEBUG: NO NEXT RESULTS ####\n";
							$trydifferent=1;
						}
					}  # if/else on whether there were matches
				if((time - $tstamp2) > 20){
#print "expiro segundo timer\n";
					return;
				}
				}  # End of $? (result of wget)

				if($? != 0){
#					print "# \$? was different from 0 (\$?== $?)\n";
					$totalqueries--;
				}
#			}while($? == 0 && $trydifferent == 0);
			}while($trydifferent == 0);

			if($#remsites < ($MAXREMOVEDSITES-1)){
				# If list is not complete, attempt to complete it
				GenDomListLevel($pdomain, \@remsites, $czone, 1, $MAXREMOVEDSITES);
			}
		} # for that adds new removed sites;
}


sub GetDomainsFromYahoo{
#		GetDomainsFromCrawler(\@domain, $czone, $level, $maxresults);	
		my($pdomain, $pzone, $plevel, $pmaxresults) = @_;
		my $czone=$pzone;

		$MAXREMOVEDSITES=25;

# Now set from the config file
#		$MAXTIMEPERIOD=10;

#		$maxtime=$MAXTIMEPERIOD;

#		if($maxtime < 5){
#			$maxtime=5;
#		}

		$MAXCNNORESULTS=125;
		$MINCNNORESULTS= 10;

		$limcnnoresults= $MAXCNNORESULTS/($plevel+1);

		if($limcnnoresults < $MINCNNORESULTS){
			$limcnnoresults= $MINCNNORESULTS;
		}

		$MAXCCOLLISIONS=150;
		$MINCCOLISSIONS= 10;
		$limccollisions= $MAXCCOLLISIONS/($plevel+1);

		if($limccollisions < $MINCCOLISSIONS){
			$limccollisions= $MINCCOLISSIONS;
		}


		# This triggers a change in search string after cumulative number of pages that resulted
		# in 100% collisions
		$cphaseonesubone= $limccollisions/3;
		if($cphaseonesubone<4){
			$cphaseonesubone=4;
		}

		# This triggers changes in subphases that should change faster
		$csubphase= $limccollisions/4;
		if($csubphase < 4){
			$csubphase=4;
		}

		$MAXBLANKPAGE= 25;
		$MINBLANKPAGE= 5;

		$limblankpage= $MAXBLANKPAGE/($plevel+1);

		if($limblankpage < $MINBLANKPAGE){
			$limblankpage= $MINBLANKPAGE;
		}


		$maxresults= $pmaxresults;

		$cur= 1;
		$nresults=0;
		$fpzeromatch=0;
		$nzeromatch=0;
		$czeromatch=0;
		$ncollisions=0;
		$ccollisions=0;
		$cnnoresults=0;
		$continue=1;
		$phase=0;
		$subphase=0;
		$heur="";
		$sitesstr="";
		$ftstr="";
		$remsites="";
		$totalqueries=0;

		do{   # Iteration for each page
			$matches=0;
			$totalmatches=0;
			$collisions=0;
			$hasnext=0;
			$noresults=0;

			# --user-agent "Mozilla/4.0"  works fine most of the time. curl || wget original headers cause trouble.
			$useragent=GetRandomUserAgent();

			@revname=`$wwwclient $wwwclient_sopt $wwwclient_tout $wwwclient_agnt"$useragent" "https://search.yahoo.com/search?p=site%3A$czone$sitesstr$ftstr$heur&b=$cur&pz=10&bct=0&xargs=0"`;

			$totalqueries++;

			if($? == 0){
				foreach $line (@revname){
					if($line=~ m/<div class="sw_next">[^<]+<\/div><\/a>/){
						$hasnext=1;
					}

					if($line=~ m/<ol id="b_results" role="main" aria-label="[^"]+"><li class="b_no">\s*<h1>/){
						$noresults=1;
						$cnnoresults++;

						if($cnnoresults >= $limcnnoresults){
							$continue=0;
						}
					}

					while($continue && $line =~ s/<h2[^>]*><a href="https?[^"\/]*\/\/([^"\/]*)//){
						$totalmatches++;
						$grabbeddomain= $1;


						if($grabbeddomain !~ m/$czone(:\d+)?/){
							#print "##### DEBUG: Discarded $grabbeddomain\n";
							next;
						}

						$matches++;
						$nzeromatch=0;
						$czeromatch=0;

						for($i=0; $i<= $#$pdomain; $i++){
							if($pdomain->[$i] eq $grabbeddomain){
								$collisions++;
								last;
							}
						}

						if($i > $#$pdomain){
							if($maxresults < 0 || ($maxresults >= 0 &&  $#$pdomain < $maxresults)){
								$pdomain->[$#$pdomain + 1]=$grabbeddomain;
								print "$grabbeddomain\n";
								$nresults++;
							}
							else{
								$continue=0;
							}
						}

						if($maxresults >= 0 && $#$pdomain >=$maxresults){
							$continue=0;
						} 
					}
				}

				#  There are a number of posible results:
				#
				# 1) We didn't get any results:
				#     a) because search engine banned us (condition: no matches, but no "no results" page)
				#        ACTION: Wait for some time, and retry with same query string and same "results page"
				#     b) Because the query string didn't yield results (condition: "no results" page)
				#        ACTION: First couple of times, wait a bit, and retry. Otherwise, change the query string. And
				#                if we hit the threshold of "no results", quit.
				# 2) We got results:
				#     a) Some were new results
				#        ACTION: Keep searching
				#     b) All were collisions
				#        ACTION: If we hit threshold, change query string. Otherwise, keep searching

				$trydifferent=0;

				if($totalmatches == 0){
					#print "#### DEBUG: GOT NO MATCHES ####\n";
					if($noresults == 1){
						#print "#### DEBUG: GOT NO RESULTS ####\n";
						if($nzeromatch == 0){
							$fpzeromatch=$cur;  # Record "when" this happened
						}

						$nzeromatch++;

						# The continue = 0 condition was set above, anyway
						if($cnnoresults > $limcnnoresults){
							# If we get "no results" after 25 pages we quit
							# print "#### Bing showed no results for our query, after retries. Quitting.... ####\n";
							$continue=0;
						}
						else{
							# Before we were trying the same query. But it does not really make sense.
							$trydifferent=1;
						}
					}
					else{
						#print "#### DEBUG: GOT BLANK PAGE ####\n";

						# If we dn't even get a page that says "no results", we assume we have been banned. Back off, and retry.
						if($nzeromatch == 0){
							$fpzeromatch=$cur;  # Record "when" this happened
							#print "# Offset: $fpzeromatch\n";
						}

						$nzeromatch++;
						$czeromatch++;

						# If it is the first time the main regex fails, chances are that bing has "banned" us. Wait a bit, and retry.
						#print "#### DEBUG: SLEEP A BIT ####\n";
						sleep(rand(($nzeromatch)*15));

						if($nzeromatch > $limblankpage || $czeromatch  > $limblankpage){
							# If we get nothing after 25 pages we quit
							#print "#### Bing has banned us. Quitting.... ####\n";
							$continue=0;
						}

						if($plevel > 0 && $nzeromatch > 2){
							$trydifferent=1;
						}
					}
				}
				else{
					# We did get search results

					# If all were collisions, count the number of collisions to exit on threshold.
					if($collisions >= $matches){
						#print "#### DEBUG: GOT ALL COLLISIONS ####\n";
						$ncollisions++;
						$ccollisions++;  # Cumulative colissions even across changes in search string

						if( ($phase >= 2 && $ccollisions > $limccollisions) || ($plevel > 0 && $phase > 0 && $ccollisions > $limccollisions)){
							# If we get nothing after 100 pages (across change of query strings), and we are in 
							# phase 2 or more, we quit
							# print "#### Couldn't get any new results. Quitting.... ####\n";
							$continue=0;
						}

						if($phase == 0){
							if($subphase == 0){
								if($ncollisions > $cphaseonesubone){
									$trydifferent=1;
								}
							}
							else{
								if($ncollisions > $csubphase){
									$trydifferent=1;
								}								
							}
						}
						elsif($phase == 1){
							if($ncollisions > $cphaseonesubone){
								$trydifferent=1;
							}
						}
						elsif($phase == 2){
							if($ncollisions > $csubphase){
								$trydifferent=1;
							}
						}
						else{
							if($ncollisions > $csubphase){
								$trydifferent=1;
							}
						}

						$cur= $cur+ $totalmatches;
					}
					else{
						$difmatch=$totalmatches-$matches;
						# Not all results were collisions. So get the next search results.
						$cur= $cur+$totalmatches;
						$ncollisions=0;
						$ccollisions=0;
						#print "### GOT NEW DATA ####\n"
					}

					# If last search didn't have "Next results", we have no other option than changing the query stringg
					if($hasnext == 0){
						#print "#### DEBUG: NO NEXT RESULTS ####\n";
						$trydifferent=1;
					}

				}  # if/else on whether there were matches

				if($trydifferent == 1){
					# subfase0: no pongo nada extra
					# subfase1: tipos de archivos
					# subfase2: elimino sites
					# subfase3: elimino sites sin www;
					# subfase4: tipos de archivo + elimino sites	
					# subfase5: tipos de archivo ´+ sites sin www

					if($subphase == 0){
						$subphase=1;
						$ftindex=0;
						$ftstr="+filetype%3A$filetypesstr[$ftindex]";
						$ncollisions=0;
						$cur=1; # Since we change the search keys, we retrieve the first page of results
					}
					elsif($subphase == 1){
						if($ftindex >= $#filetypesstr){
							$ftstr="";
							$ftindex=0;
							$subphase=2;
							undef @remsites;
							$sitesstr=GetRemSitesString($pdomain, $czone, 0);
						}
						else{
							$ftindex++;
							$ftstr="+filetype%3A$filetypesstr[$ftindex]";
						}

						$ncollisions=0;
						$cur=1; # Since we change the search keys, we retrieve the first page of results
					}
					elsif($subphase == 2){
						$subphase=3;
						undef @remsites;
						$sitesstr=GetRemSitesString($pdomain, $czone, 1);
						$ncollisions=0;
						$cur=1; # Since we change the search keys, we retrieve the first page of results
					}
					elsif($subphase == 3){
						$subphase=4;
						undef @remsites;
						$sitesstr=GetRemSitesString($pdomain, $czone, 0);
						$ftindex=0;
						$ftstr="+filetype%3A$filetypesstr[$ftindex]";
						$ncollisions=0;
						$cur=1; # Since we change the search keys, we retrieve the first page of results
					}
					elsif($subphase == 4){
						if($ftindex >= $#filetypesstr){
							$subphase=5;
							$ftindex=0;
							$ftstr="+filetype%3A$filetypesstr[$ftindex]";
							undef @remsites;
							$sitesstr=GetRemSitesString($pdomain, $czone, 1);
						}
						else{
							$ftindex++;
							$ftstr="+filetype%3A$filetypesstr[$ftindex]";
						}

						$ncollisions=0;
						$cur=1; # Since we change the search keys, we retrieve the first page of results
					}
					elsif($subphase ==5){
						if($ftindex < $#filetypesstr){
							$ftindex++;
							$ftstr="+filetype%3A$filetypesstr[$ftindex]";
							$ncollisions=0;
							$cur=1; # Since we change the search keys, we retrieve the first page of results
						}
						else{
							$ftstr="";
							$ftindex=0;
							$sitesstr="";
							$subphase=0;

							if($phase == 0){
								# print "#### Phase 1  ####\n";
								$phase=1;
								$letterkey="a";
								$heur="+$letterkey";
								$ncollisions=0;
								$cur=1; # Since we change the search keys, we retrieve the first page of results
							}
							elsif($phase == 1){
								# In phase 1 we search for the domain name and one letter
								if($letterkey eq "z"){
									$letterkey="a";
									$numkey="0";
									$heur="+$numkey";
									$ncollisions=0;
									$phase=2;
									$cur=1; # Since we change the search keys, we retrieve the first page of results
									# print "#### Phase 2  ####\n";
								}
								else{
									$letterkey++;
									$ncollisions=0;
									$heur="+$letterkey";
									$cur=1; # Since we change the search keys, we retrieve the first page of results
								}
							}
							elsif($phase == 2){
								# In phase 2 we search for the domain name and a number

								if($numkey >= 10001){
									$letterkey="a";
									$numkey=0;
									$heur="+$letterkey+$numkey";
									$ncollisions=0;
									$phase=3;
									$cur=1; # Since we change the search keys, we retrieve the first page of results
									# print "#### Phase 3  ####\n";
								}
								else{
									$numkey++;
									$ncollisions=0;
									$cur=1; # Since we change the search keys, we retrieve the first page of results
									$heur="+$numkey";
								}
							}
							else{
								# In phase #3, we search for the domainname, plus a letter and a number
								if($numkey > 10000){
									if($letterkey eq "z"){
										# print "#### Tried my best. Quitting.... ####\n";
										$continue=0;
									}
									else{
										$letterkey++;
										$numkey=0;
										$cur=1; # Since we change the search keys, we retrieve the first page of results
										$heur="+$letterkey+$numkey";
										$ncollisions=0;
									}
								}
								else{
									$numkey++;
									$ncollisions=0;
									$cur=1; # Since we change the search keys, we retrieve the first page of results
									$heur="+$letterkey+$numkey";
								}
							} #end oh phase 
						} # end of else in subphase 5

					} #end of subphase

				}  # End of "trydifferent"
			}  # End of $? (result of wget)
		}while($? == 0 && $continue == 1);


		if($maxresults > 0 && $#$pdomain >= $maxresults){
			return;
		}


		# We will now exclude next level domains

		undef @remsites;

		GenDomListLevel($pdomain, \@remsites, $czone, 1, $MAXREMOVEDSITES);

		$continue=1;

		for($jjj= 0; $jjj <= $#remsites && $continue == 1; $jjj++){
			$cur= 1;

			$sitesstr= GenRemListStr(\@remsites, $jjj);
			$trydifferent=0;

			do{   # Iteration for each page
				$matches=0;
				$totalmatches=0;
				$collisions=0;
				$hasnext=0;
				$noresults=0;

				$useragent=GetRandomUserAgent();

				@revname=`$wwwclient $wwwclient_sopt $wwwclient_tout $wwwclient_agnt"$useragent" "https://search.yahoo.com/search?p=site%3A$czone$sitesstr$ftstr$heur&b=$cur&pz=10&bct=0&xargs=0"`;

				$totalqueries++;

				if($? == 0){
					foreach $line (@revname){
						if($continue == 0){
							last;
						}

						if($line=~ m/<div class="sw_next">[^<]+<\/div><\/a>/){
							$hasnext=1;
						}

						if($line=~ m/<ol id="b_results" role="main" aria-label="[^"]+"><li class="b_no">\s*<h1>/){
							$noresults=1;
						}

						while($continue && $line =~ s/<h2[^>]*><a href="https?[^"\/]*\/\/([^"\/]*)//){
							$totalmatches++;
							$grabbeddomain= $1;

							if($grabbeddomain !~ m/$czone(:\d+)?/){
								next;
							}

							$matches++;
							$nzeromatch=0;

							for($k=0; $k<= $#$pdomain; $k++){
								if($pdomain->[$k] eq $grabbeddomain){
									$collisions++;
									last;
								}
							}


							if($k > $#$pdomain){
								if($maxresults < 0 || ($maxresults >= 0 &&  $#$pdomain < $maxresults)){
									$pdomain->[$#$pdomain + 1]=$grabbeddomain;
									print "$grabbeddomain\n";
									$nresults++;
								}
								else{
									$continue=0;
								}
							}

							if($maxresults >= 0 && $#$pdomain>=$maxresults){
								$continue=0;
							} 
						} # while() that reads lines
					}  # foreach $line

					$trydifferent=0;

					if($totalmatches == 0){
						if($noresults == 1){
							$trydifferent=1;
						}
						else{
							#print "#### DEBUG: GOT BLANK PAGE ####\n";
							$nzeromatch++;

							# If it is the first time the main regex fails, chances are that bing has "banned" us. Wait a bit, and retry.
							#print "#### DEBUG: SLEEP A BIT ####\n";
							sleep(rand(($nzeromatch)*20));

							if($nzeromatch > 5){
								#	print "#### Bing has banned us. Quitting.... ####\n";
								$trydifferent=1;
							}
						}
					}
					else{
						# We did get search results

						# If all were collisions, count the number of collisions to exit on threshold.
						if($collisions >= $matches){
							#print "#### DEBUG: GOT ALL COLLISIONS ####\n";
							$ncollisions++;

							if($ncollisions >= 10){
								$trydifferent=1;
							} 

							$cur= $cur+ $totalmatches;
						}
						else{
							$difmatch=$totalmatches-$matches;
							# Not all results were collisions. So get the next search results.
							$cur= $cur+$totalmatches;
							$ncollisions=0;
						}

						# If last search didn't have "Next results", we have no other option than changing the query stringg
						if($hasnext == 0){
							#print "#### DEBUG: NO NEXT RESULTS ####\n";
							$trydifferent=1;
						}
					}  # if/else on whether there were matches
				}  # End of $? (result of wget)
			}while($? == 0 && $trydifferent == 0);

			if($#remsites < ($MAXREMOVEDSITES-1)){
				# If list is not complete, attempt to complete it
				GenDomListLevel($pdomain, \@remsites, $czone, 1, $MAXREMOVEDSITES);
			}
		} # for that adds new removed sites;
}


sub GetDomainsFromTeoma{
#		GetDomainsFromCrawler(\@domain, $czone, $level, $maxresults);	
		my($pdomain, $pzone, $plevel, $pmaxresults) = @_;
		my $czone=$pzone;

		$MAXREMOVEDSITES=25;

# Now set from the config file
#		$MAXTIMEPERIOD=10;

#		$maxtime=$MAXTIMEPERIOD;

#		if($maxtime < 5){
#			$maxtime=5;
#		}

		$MAXCNNORESULTS=125;
		$MINCNNORESULTS= 10;

		$limcnnoresults= $MAXCNNORESULTS/($plevel+1);

		if($limcnnoresults < $MINCNNORESULTS){
			$limcnnoresults= $MINCNNORESULTS;
		}

		$MAXCCOLLISIONS=150;
		$MINCCOLISSIONS= 10;
		$limccollisions= $MAXCCOLLISIONS/($plevel+1);

		if($limccollisions < $MINCCOLISSIONS){
			$limccollisions= $MINCCOLISSIONS;
		}


		# This triggers a change in search string after cumulative number of pages that resulted
		# in 100% collisions
		$cphaseonesubone= $limccollisions/3;
		if($cphaseonesubone<4){
			$cphaseonesubone=4;
		}

		# This triggers changes in subphases that should change faster
		$csubphase= $limccollisions/4;
		if($csubphase < 4){
			$csubphase=4;
		}

		$MAXBLANKPAGE= 25;
		$MINBLANKPAGE= 5;

		$limblankpage= $MAXBLANKPAGE/($plevel+1);

		if($limblankpage < $MINBLANKPAGE){
			$limblankpage= $MINBLANKPAGE;
		}


		$maxresults= $pmaxresults;

		$cur= 1;
		$nresults=0;
		$fpzeromatch=0;
		$nzeromatch=0;
		$czeromatch=0;
		$ncollisions=0;
		$ccollisions=0;
		$cnnoresults=0;
		$continue=1;
		$phase=0;
		$subphase=0;
		$heur="";
		$sitesstr="";
		$ftstr="";
		$remsites="";
		$totalqueries=0;
		$naccessdenied=0;

		do{   # Iteration for each page
			$accessdenied=0;
			$matches=0;
			$totalmatches=0;
			$collisions=0;
			$hasnext=0;

			# We assume there were no results. If there are, we reset $noresults to 0
			$noresults=1;

			# --user-agent "Mozilla/4.0"  works fine most of the time. curl || wget original headers cause trouble.
			$useragent=GetRandomUserAgent();

			@revname=`$wwwclient $wwwclient_sopt $wwwclient_tout $wwwclient_agnt"$useragent" "https://www.teoma.com/web?q=site%3A$czone$sitesstr$ftstr$heur&page=$cur"`;

			$totalqueries++;

			if($? == 0){
				foreach $line (@revname){
					if($line=~ m/<div id="accessDenied">/){
						$accessdenied=1;
					}

					if($line=~ m/<div class="pagination-next">/){
						$hasnext=1;
					}

					if($line=~ m/<cite\s+class="algo-display-url">https?:\/\/([^\/<]+)/){
						$noresults=0;
						$totalmatches++;
						$grabbeddomain= $1;

						if($grabbeddomain !~ m/$czone(:\d+)?/){
							#print "##### DEBUG: Discarded $grabbeddomain\n";
							next;
						}

						$matches++;
						$nzeromatch=0;
						$czeromatch=0;

						for($i=0; $i<= $#$pdomain; $i++){
							if($pdomain->[$i] eq $grabbeddomain){
								$collisions++;
								last;
							}
						}

						if($i > $#$pdomain){
							if($maxresults < 0 || ($maxresults >= 0 &&  $#$pdomain < $maxresults)){
								$pdomain->[$#$pdomain + 1]=$grabbeddomain;
								print "$grabbeddomain\n";
								$nresults++;
							}
							else{
								$continue=0;
								last;
							}
						}

						if($maxresults >= 0 && $#$pdomain >=$maxresults){
							$continue=0;
							last;
						} 
					}
				}

				#  There are a number of posible results:
				#
				# 1) We didn't get any results:
				#     a) because search engine banned us (condition: no matches, but no "no results" page)
				#        ACTION: Wait for some time, and retry with same query string and same "results page"
				#     b) Because the query string didn't yield results (condition: "no results" page)
				#        ACTION: First couple of times, wait a bit, and retry. Otherwise, change the query string. And
				#                if we hit the threshold of "no results", quit.
				# 2) We got results:
				#     a) Some were new results
				#        ACTION: Keep searching
				#     b) All were collisions
				#        ACTION: If we hit threshold, change query string. Otherwise, keep searching

				$trydifferent=0;

				# totalmatches == 0 is the same as noresults=1
				if($totalmatches == 0){
					if($accessdenied == 1){
						$naccessdenied++;

						if($naccessdenied > $limblankpage){
							print "# Teoma has banned us (page results: $cur). Quitting....\n";
							exit;
						}

						sleep(60+$naccessdenied * 60);
					}
					else{

						#print "#### DEBUG: GOT NO RESULTS ####\n";
						if($nzeromatch == 0){
							$fpzeromatch=$cur;  # Record "when" this happened
						}

						$nzeromatch++;

						# The continue = 0 condition was set above, anyway
						if($cnnoresults > $limcnnoresults){
							# If we get "no results" after 25 pages we quit
							# print "#### Bing showed no results for our query, after retries. Quitting.... ####\n";
							$continue=0;
						}
						else{
							# Before we were trying the same query. But it does not really make sense.
							$trydifferent=1;
						}
					}
				}
				else{
					# We did get search results
					$accessdenied=0;

					# If all were collisions, count the number of collisions to exit on threshold.
					if($collisions >= $matches){
						#print "#### DEBUG: GOT ALL COLLISIONS ####\n";
						$ncollisions++;
						$ccollisions++;  # Cumulative colissions even across changes in search string

						if( ($phase >= 2 && $ccollisions > $limccollisions) || ($plevel > 0 && $phase > 0 && $ccollisions > $limccollisions)){
							# If we get nothing after 100 pages (across change of query strings), and we are in 
							# phase 2 or more, we quit
							# print "#### Couldn't get any new results. Quitting.... ####\n";
							$continue=0;
						}

						if($phase == 0){
							if($subphase == 0){
								if($ncollisions > $cphaseonesubone){
									$trydifferent=1;
								}
							}
							else{
								if($ncollisions > $csubphase){
									$trydifferent=1;
								}								
							}
						}
						elsif($phase == 1){
							if($ncollisions > $cphaseonesubone){
								$trydifferent=1;
							}
						}
						elsif($phase == 2){
							if($ncollisions > $csubphase){
								$trydifferent=1;
							}
						}
						else{
							if($ncollisions > $csubphase){
								$trydifferent=1;
							}
						}

						$cur++;
					}
					else{
						$difmatch=$totalmatches-$matches;
						# Not all results were collisions. So get the next search results.
						$cur++;
						$ncollisions=0;
						$ccollisions=0;
						#print "### GOT NEW DATA ####\n"
					}

					# If last search didn't have "Next results", we have no other option than changing the query stringg
					if($hasnext == 0){
						#print "#### DEBUG: NO NEXT RESULTS ####\n";
						$trydifferent=1;
					}

				}  # if/else on whether there were matches

				if($trydifferent == 1){
					# subfase0: no pongo nada extra
					# subfase1: tipos de archivos
					# subfase2: elimino sites
					# subfase3: elimino sites sin www;
					# subfase4: tipos de archivo + elimino sites	
					# subfase5: tipos de archivo ´+ sites sin www

					if($subphase == 0){
						$subphase=1;
						$ftindex=0;
						$ftstr="+filetype%3A$filetypesstr[$ftindex]";
						$ncollisions=0;
						$cur=1; # Since we change the search keys, we retrieve the first page of results
					}
					elsif($subphase == 1){
						if($ftindex >= $#filetypesstr){
							$ftstr="";
							$ftindex=0;
							$subphase=2;
							undef @remsites;
							$sitesstr=GetRemSitesString($pdomain, $czone, 0);
						}
						else{
							$ftindex++;
							$ftstr="+filetype%3A$filetypesstr[$ftindex]";
						}

						$ncollisions=0;
						$cur=1; # Since we change the search keys, we retrieve the first page of results
					}
					elsif($subphase == 2){
						$subphase=3;
						undef @remsites;
						$sitesstr=GetRemSitesString($pdomain, $czone, 1);
						$ncollisions=0;
						$cur=1; # Since we change the search keys, we retrieve the first page of results
					}
					elsif($subphase == 3){
						$subphase=4;
						undef @remsites;
						$sitesstr=GetRemSitesString($pdomain, $czone, 0);
						$ftindex=0;
						$ftstr="+filetype%3A$filetypesstr[$ftindex]";
						$ncollisions=0;
						$cur=1; # Since we change the search keys, we retrieve the first page of results
					}
					elsif($subphase == 4){
						if($ftindex >= $#filetypesstr){
							$subphase=5;
							$ftindex=0;
							$ftstr="+filetype%3A$filetypesstr[$ftindex]";
							undef @remsites;
							$sitesstr=GetRemSitesString($pdomain, $czone, 1);
						}
						else{
							$ftindex++;
							$ftstr="+filetype%3A$filetypesstr[$ftindex]";
						}

						$ncollisions=0;
						$cur=1; # Since we change the search keys, we retrieve the first page of results
					}
					elsif($subphase ==5){
						if($ftindex < $#filetypesstr){
							$ftindex++;
							$ftstr="+filetype%3A$filetypesstr[$ftindex]";
							$ncollisions=0;
							$cur=1; # Since we change the search keys, we retrieve the first page of results
						}
						else{
							$ftstr="";
							$ftindex=0;
							$sitesstr="";
							$subphase=0;

							if($phase == 0){
								# print "#### Phase 1  ####\n";
								$phase=1;
								$letterkey="a";
								$heur="+$letterkey";
								$ncollisions=0;
								$cur=1; # Since we change the search keys, we retrieve the first page of results
							}
							elsif($phase == 1){
								# In phase 1 we search for the domain name and one letter
								if($letterkey eq "z"){
									$letterkey="a";
									$numkey="0";
									$heur="+$numkey";
									$ncollisions=0;
									$phase=2;
									$cur=1; # Since we change the search keys, we retrieve the first page of results
									# print "#### Phase 2  ####\n";
								}
								else{
									$letterkey++;
									$ncollisions=0;
									$heur="+$letterkey";
									$cur=1; # Since we change the search keys, we retrieve the first page of results
								}
							}
							elsif($phase == 2){
								# In phase 2 we search for the domain name and a number

								if($numkey >= 10001){
									$letterkey="a";
									$numkey=0;
									$heur="+$letterkey+$numkey";
									$ncollisions=0;
									$phase=3;
									$cur=1; # Since we change the search keys, we retrieve the first page of results
									# print "#### Phase 3  ####\n";
								}
								else{
									$numkey++;
									$ncollisions=0;
									$cur=1; # Since we change the search keys, we retrieve the first page of results
									$heur="+$numkey";
								}
							}
							else{
								# In phase #3, we search for the domainname, plus a letter and a number
								if($numkey > 10000){
									if($letterkey eq "z"){
										# print "#### Tried my best. Quitting.... ####\n";
										$continue=0;
									}
									else{
										$letterkey++;
										$numkey=0;
										$cur=1; # Since we change the search keys, we retrieve the first page of results
										$heur="+$letterkey+$numkey";
										$ncollisions=0;
									}
								}
								else{
									$numkey++;
									$ncollisions=0;
									$cur=1; # Since we change the search keys, we retrieve the first page of results
									$heur="+$letterkey+$numkey";
								}
							} #end oh phase 
						} # end of else in subphase 5

					} #end of subphase

				}  # End of "trydifferent"
			}  # End of $? (result of wget)


			# XXX Teoma will block us if we query too fast. Still need to tune this to fly low enough
			sleep($cur + $cur * 5);
		}while($? == 0 && $continue == 1);


		if($maxresults >= 0 && $#$pdomain >= $maxresults){
			return;
		}


		# We will now exclude next level domains

		undef @remsites;

		GenDomListLevel($pdomain, \@remsites, $czone, 1, $MAXREMOVEDSITES);

		$continue=1;

		for($jjj= 0; $jjj <= $#remsites && $continue == 1; $jjj++){
			$cur= 1;

			$sitesstr= GenRemListStr(\@remsites, $jjj);
			$trydifferent=0;

			do{   # Iteration for each page
				$matches=0;
				$totalmatches=0;
				$collisions=0;
				$hasnext=0;
				$noresults=1;
				$accessdenied=0;

				$useragent=GetRandomUserAgent();

				@revname=`$wwwclient $wwwclient_sopt $wwwclient_tout $wwwclient_agnt"$useragent" "https://www.teoma.com/web?q=site%3A$czone$sitesstr&page=$cur"`;

				$totalqueries++;

				if($? == 0){
					foreach $line (@revname){
						if($continue == 0){
							last;
						}

						if($line=~ m/<div id="accessDenied">/){
							$accessdenied=1;
						}

						if($line=~ m/<div class="pagination-next">/){
							$hasnext=1;
						}

						if($line=~ m/<cite\s+class="algo-display-url">https?:\/\/([^\/<]+)/){
							$noresults=0;;
							$totalmatches++;
							$grabbeddomain= $1;

							if($grabbeddomain !~ m/$czone(:\d+)?/){
								next;
							}

							$matches++;
							$nzeromatch=0;

							for($k=0; $k<= $#$pdomain; $k++){
								if($pdomain->[$k] eq $grabbeddomain){
									$collisions++;
									last;
								}
							}


							if($k > $#$pdomain){
								if($maxresults < 0 || ($maxresults >= 0 &&  $#$pdomain < $maxresults)){
									$pdomain->[$#$pdomain + 1]=$grabbeddomain;
									print "$grabbeddomain\n";
									$nresults++;
								}
								else{
									$continue=0;
								}
							}

							if($maxresults >= 0 && $#$pdomain>=$maxresults){
								$continue=0;
							} 
						} # while() that reads lines
					}  # foreach $line

					$trydifferent=0;

					if($totalmatches == 0){
						if($accessdenied == 1){
							$naccessdenied++;

							# XXX: Should keep track of # of queries
							if($naccessdenied > $limblankpage){
								print "# Teoma has banned us (page results: $cur). Quitting....\n";
								exit;
							}

							sleep(60+$naccessdenied * 60);
						}
						else{
							$trydifferent=1;
						}
					}
					else{
						# We did get search results
						$accessdenied=0;
						# If all were collisions, count the number of collisions to exit on threshold.
						if($collisions >= $matches){
							#print "#### DEBUG: GOT ALL COLLISIONS ####\n";
							$ncollisions++;

							if($ncollisions >= 10){
								$trydifferent=1;
							} 

							$cur++;
						}
						else{
							$difmatch=$totalmatches-$matches;
							# Not all results were collisions. So get the next search results.
							$cur++;
							$ncollisions=0;
						}

						# If last search didn't have "Next results", we have no other option than changing the query stringg
						if($hasnext == 0){
							#print "#### DEBUG: NO NEXT RESULTS ####\n";
							$trydifferent=1;
						}
					}  # if/else on whether there were matches
				}  # End of $? (result of wget)

				# XXX Teoma will block us if we query too fast. Still need to tune this to fly low enough
				sleep($cur + $cur * 5);
			}while($? == 0 && $trydifferent == 0);

			if($#remsites < ($MAXREMOVEDSITES-1)){
				# If list is not complete, attempt to complete it
				GenDomListLevel($pdomain, \@remsites, $czone, 1, $MAXREMOVEDSITES);
			}
		} # for that adds new removed sites;
}




sub GetDomainsFromWebcrawler{
#		GetDomainsFromCrawler(\@domain, $czone, $level, $maxresults);	
		my($pdomain, $pzone, $plevel, $pmaxresults) = @_;
		my $czone=$pzone;

		$MAXREMOVEDSITES=25;

# Now set from the config file
#		$MAXTIMEPERIOD=10;

#		$maxtime=$MAXTIMEPERIOD;

#		if($maxtime < 5){
#			$maxtime=5;
#		}

		$MAXCNNORESULTS=125;
		$MINCNNORESULTS= 10;

		$limcnnoresults= $MAXCNNORESULTS/($plevel+1);

		if($limcnnoresults < $MINCNNORESULTS){
			$limcnnoresults= $MINCNNORESULTS;
		}

		$MAXCCOLLISIONS=150;
		$MINCCOLISSIONS= 10;
		$limccollisions= $MAXCCOLLISIONS/($plevel+1);

		if($limccollisions < $MINCCOLISSIONS){
			$limccollisions= $MINCCOLISSIONS;
		}


		# This triggers a change in search string after cumulative number of pages that resulted
		# in 100% collisions
		$cphaseonesubone= $limccollisions/3;
		if($cphaseonesubone<4){
			$cphaseonesubone=4;
		}

		# This triggers changes in subphases that should change faster
		$csubphase= $limccollisions/4;
		if($csubphase < 4){
			$csubphase=4;
		}

		$MAXBLANKPAGE= 25;
		$MINBLANKPAGE= 5;

		$limblankpage= $MAXBLANKPAGE/($plevel+1);

		if($limblankpage < $MINBLANKPAGE){
			$limblankpage= $MINBLANKPAGE;
		}


		$maxresults= $pmaxresults;

		$cur= 1;
		$nresults=0;
		$fpzeromatch=0;
		$nzeromatch=0;
		$czeromatch=0;
		$ncollisions=0;
		$ccollisions=0;
		$cnnoresults=0;
		$continue=1;
		$phase=0;
		$subphase=0;
		$heur="";
		$sitesstr="";
		$ftstr="";
		$remsites="";
		$totalqueries=0;
		$naccessdenied=0;

		do{   # Iteration for each page
			$accessdenied=0;
			$matches=0;
			$totalmatches=0;
			$collisions=0;
			$hasnext=0;

			# We assume there were no results. If there are, we reset $noresults to 0
			$noresults=1;

			# --user-agent "Mozilla/4.0"  works fine most of the time. curl || wget original headers cause trouble.
			$useragent=GetRandomUserAgent();

			if($cur == 1){
				@revname=`$wwwclient $wwwclient_sopt $wwwclient_tout $wwwclient_agnt"$useragent" "http://www.webcrawler.com/serp?q=site%3A$czone$sitesstr$ftstr$heur"`;
			}
			else{
				@revname=`$wwwclient $wwwclient_sopt $wwwclient_tout $wwwclient_agnt"$useragent" "http://www.webcrawler.com/$nextpage"`;
			}


			$totalqueries++;

			if($? == 0){
				foreach $line (@revname){

					if($line=~ m/<div id="accessDenied">/){
						$accessdenied=1;
					}

					if($line=~ m/<a class="num next" href="([^"]+)">[^<]+<\/a>/){
						$hasnext=1;
						$nextpage=$1;
#print "#DEBUG. Prox pag: $nextpage\n";
					}

#if($line=~ m/<a class="title"/){
#print "#DEBUG: Principio de regex ok!\n";
#}

#if($line=~ m/<a class="title" href="([^"]+)"/){
#print "#DEBUG: Matchie esto: $1\n";
#}

					if($line=~ m/<a class="title" href="https?:\/\/([^<\/"]+)"/){
						$noresults=0;
						$totalmatches++;
						$grabbeddomain= $1;

						if($grabbeddomain !~ m/$czone(:\d+)?/){
							#print "##### DEBUG: Discarded $grabbeddomain\n";
							next;
						}

						$matches++;
						$nzeromatch=0;
						$czeromatch=0;
#print "# DEBUG: Lei dominio $grabbeddomain\n";

						for($i=0; $i<= $#$pdomain; $i++){
							if($pdomain->[$i] eq $grabbeddomain){
								$collisions++;
								last;
							}
						}

						if($i > $#$pdomain){
							if($maxresults < 0 || ($maxresults >= 0 &&  $#$pdomain < $maxresults)){
								$pdomain->[$#$pdomain + 1]=$grabbeddomain;
								print "$grabbeddomain\n";
								$nresults++;
							}
							else{
								$continue=0;
								last;
							}
						}

						if($maxresults >= 0 && $#$pdomain >=$maxresults){
							$continue=0;
							last;
						} 
					}
				}

				#  There are a number of posible results:
				#
				# 1) We didn't get any results:
				#     a) because search engine banned us (condition: no matches, but no "no results" page)
				#        ACTION: Wait for some time, and retry with same query string and same "results page"
				#     b) Because the query string didn't yield results (condition: "no results" page)
				#        ACTION: First couple of times, wait a bit, and retry. Otherwise, change the query string. And
				#                if we hit the threshold of "no results", quit.
				# 2) We got results:
				#     a) Some were new results
				#        ACTION: Keep searching
				#     b) All were collisions
				#        ACTION: If we hit threshold, change query string. Otherwise, keep searching

				$trydifferent=0;

				# totalmatches == 0 is the same as noresults=1
				if($totalmatches == 0){
					if($accessdenied == 1){
						$naccessdenied++;

						if($naccessdenied > $limblankpage){
							print "# Webcrawler has banned us (page results: $cur). Quitting....\n";
							exit;
						}

						sleep(60+$naccessdenied * 60);
					}
					else{

						#print "#### DEBUG: GOT NO RESULTS ####\n";
						if($nzeromatch == 0){
							$fpzeromatch=$cur;  # Record "when" this happened
						}

						$nzeromatch++;

						# The continue = 0 condition was set above, anyway
						if($cnnoresults > $limcnnoresults){
							# If we get "no results" after 25 pages we quit
							# print "#### Bing showed no results for our query, after retries. Quitting.... ####\n";
							$continue=0;
						}
						else{
							# Before we were trying the same query. But it does not really make sense.
							$trydifferent=1;
						}
					}
				}
				else{
					# We did get search results

					# If all were collisions, count the number of collisions to exit on threshold.
					if($collisions >= $matches){
						#print "#### DEBUG: GOT ALL COLLISIONS ####\n";
						$ncollisions++;
						$ccollisions++;  # Cumulative colissions even across changes in search string

						if( ($phase >= 2 && $ccollisions > $limccollisions) || ($plevel > 0 && $phase > 0 && $ccollisions > $limccollisions)){
							# If we get nothing after 100 pages (across change of query strings), and we are in 
							# phase 2 or more, we quit
							# print "#### Couldn't get any new results. Quitting.... ####\n";
							$continue=0;
						}

						if($phase == 0){
							if($subphase == 0){
								if($ncollisions > $cphaseonesubone){
									$trydifferent=1;
								}
							}
							else{
								if($ncollisions > $csubphase){
									$trydifferent=1;
								}								
							}
						}
						elsif($phase == 1){
							if($ncollisions > $cphaseonesubone){
								$trydifferent=1;
							}
						}
						elsif($phase == 2){
							if($ncollisions > $csubphase){
								$trydifferent=1;
							}
						}
						else{
							if($ncollisions > $csubphase){
								$trydifferent=1;
							}
						}

						$cur++;
					}
					else{
						$difmatch=$totalmatches-$matches;
						# Not all results were collisions. So get the next search results.
						$cur++;
						$ncollisions=0;
						$ccollisions=0;
						#print "### GOT NEW DATA ####\n"
					}

					# If last search didn't have "Next results", we have no other option than changing the query stringg
					if($hasnext == 0){
						#print "#### DEBUG: NO NEXT RESULTS ####\n";
						$trydifferent=1;
					}

				}  # if/else on whether there were matches

				if($trydifferent == 1){
					# subfase0: no pongo nada extra
					# subfase1: tipos de archivos
					# subfase2: elimino sites
					# subfase3: elimino sites sin www;
					# subfase4: tipos de archivo + elimino sites	
					# subfase5: tipos de archivo ´+ sites sin www

					if($subphase == 0){
						$subphase=1;
						$ftindex=0;
						$ftstr="+filetype%3A$filetypesstr[$ftindex]";
						$ncollisions=0;
						$cur=1; # Since we change the search keys, we retrieve the first page of results
					}
					elsif($subphase == 1){
						if($ftindex >= $#filetypesstr){
							$ftstr="";
							$ftindex=0;
							$subphase=2;
							undef @remsites;
							$sitesstr=GetRemSitesString($pdomain, $czone, 0);
						}
						else{
							$ftindex++;
							$ftstr="+filetype%3A$filetypesstr[$ftindex]";
						}

						$ncollisions=0;
						$cur=1; # Since we change the search keys, we retrieve the first page of results
					}
					elsif($subphase == 2){
						$subphase=3;
						undef @remsites;
						$sitesstr=GetRemSitesString($pdomain, $czone, 1);
						$ncollisions=0;
						$cur=1; # Since we change the search keys, we retrieve the first page of results
					}
					elsif($subphase == 3){
						$subphase=4;
						undef @remsites;
						$sitesstr=GetRemSitesString($pdomain, $czone, 0);
						$ftindex=0;
						$ftstr="+filetype%3A$filetypesstr[$ftindex]";
						$ncollisions=0;
						$cur=1; # Since we change the search keys, we retrieve the first page of results
					}
					elsif($subphase == 4){
						if($ftindex >= $#filetypesstr){
							$subphase=5;
							$ftindex=0;
							$ftstr="+filetype%3A$filetypesstr[$ftindex]";
							undef @remsites;
							$sitesstr=GetRemSitesString($pdomain, $czone, 1);
						}
						else{
							$ftindex++;
							$ftstr="+filetype%3A$filetypesstr[$ftindex]";
						}

						$ncollisions=0;
						$cur=1; # Since we change the search keys, we retrieve the first page of results
					}
					elsif($subphase ==5){
						if($ftindex < $#filetypesstr){
							$ftindex++;
							$ftstr="+filetype%3A$filetypesstr[$ftindex]";
							$ncollisions=0;
							$cur=1; # Since we change the search keys, we retrieve the first page of results
						}
						else{
							$ftstr="";
							$ftindex=0;
							$sitesstr="";
							$subphase=0;

							if($phase == 0){
								# print "#### Phase 1  ####\n";
								$phase=1;
								$letterkey="a";
								$heur="+$letterkey";
								$ncollisions=0;
								$cur=1; # Since we change the search keys, we retrieve the first page of results
							}
							elsif($phase == 1){
								# In phase 1 we search for the domain name and one letter
								if($letterkey eq "z"){
									$letterkey="a";
									$numkey="0";
									$heur="+$numkey";
									$ncollisions=0;
									$phase=2;
									$cur=1; # Since we change the search keys, we retrieve the first page of results
									# print "#### Phase 2  ####\n";
								}
								else{
									$letterkey++;
									$ncollisions=0;
									$heur="+$letterkey";
									$cur=1; # Since we change the search keys, we retrieve the first page of results
								}
							}
							elsif($phase == 2){
								# In phase 2 we search for the domain name and a number

								if($numkey >= 10001){
									$letterkey="a";
									$numkey=0;
									$heur="+$letterkey+$numkey";
									$ncollisions=0;
									$phase=3;
									$cur=1; # Since we change the search keys, we retrieve the first page of results
									# print "#### Phase 3  ####\n";
								}
								else{
									$numkey++;
									$ncollisions=0;
									$cur=1; # Since we change the search keys, we retrieve the first page of results
									$heur="+$numkey";
								}
							}
							else{
								# In phase #3, we search for the domainname, plus a letter and a number
								if($numkey > 10000){
									if($letterkey eq "z"){
										# print "#### Tried my best. Quitting.... ####\n";
										$continue=0;
									}
									else{
										$letterkey++;
										$numkey=0;
										$cur=1; # Since we change the search keys, we retrieve the first page of results
										$heur="+$letterkey+$numkey";
										$ncollisions=0;
									}
								}
								else{
									$numkey++;
									$ncollisions=0;
									$cur=1; # Since we change the search keys, we retrieve the first page of results
									$heur="+$letterkey+$numkey";
								}
							} #end oh phase 
						} # end of else in subphase 5

					} #end of subphase

				}  # End of "trydifferent"
			}  # End of $? (result of wget)


			# XXX Teoma will block us if we query too fast. Still need to tune this to fly low enough
			sleep($cur);
		}while($? == 0 && $continue == 1);


		if($maxresults >= 0 && $#$pdomain >= $maxresults){
			return;
		}


		# We will now exclude next level domains

		undef @remsites;

		GenDomListLevel($pdomain, \@remsites, $czone, 1, $MAXREMOVEDSITES);

		$continue=1;

		for($jjj= 0; $jjj <= $#remsites && $continue == 1; $jjj++){
			$cur= 1;

			$sitesstr= GenRemListStr(\@remsites, $jjj);
			$trydifferent=0;

			do{   # Iteration for each page
				$matches=0;
				$totalmatches=0;
				$collisions=0;
				$hasnext=0;
				$noresults=1;
				$accessdenied=0;

				$useragent=GetRandomUserAgent();

				if($cur == 1){
					@revname=`$wwwclient $wwwclient_sopt $wwwclient_tout $wwwclient_agnt"$useragent" "http://www.webcrawler.com/serp?q=site%3A$czone$sitesstr$ftstr$heur"`;
				}
				else{
					@revname=`$wwwclient $wwwclient_sopt $wwwclient_tout $wwwclient_agnt"$useragent" "http://www.webcrawler.com/$nextpage"`;
				}

				$totalqueries++;

				if($? == 0){
					foreach $line (@revname){
						if($continue == 0){
							last;
						}

						if($line=~ m/<div id="accessDenied">/){
							$accessdenied=1;
						}

						if($line=~ m/<a class="num next" href="([^"]+)">[^<]+<\/a>/){
							$hasnext=1;
							$nextpage=$1;
						}

						if($line=~ m/<a class="title" href="https?:\/\/([^<\/"]+)"/){
#print "# DEBUG: MATCHIE RESULTADO: $1\n";
							$noresults=0;;
							$totalmatches++;
							$grabbeddomain= $1;

							if($grabbeddomain !~ m/$czone(:\d+)?/){
								next;
							}

							$matches++;
							$nzeromatch=0;

							for($k=0; $k<= $#$pdomain; $k++){
								if($pdomain->[$k] eq $grabbeddomain){
									$collisions++;
									last;
								}
							}


							if($k > $#$pdomain){
								if($maxresults < 0 || ($maxresults >= 0 &&  $#$pdomain < $maxresults)){
									$pdomain->[$#$pdomain + 1]=$grabbeddomain;
									print "$grabbeddomain\n";
									$nresults++;
								}
								else{
									$continue=0;
								}
							}

							if($maxresults >= 0 && $#$pdomain>=$maxresults){
								$continue=0;
							} 
						} # while() that reads lines
					}  # foreach $line

					$trydifferent=0;

					if($totalmatches == 0){
						if($accessdenied == 1){
							$naccessdenied++;

							# XXX: Should keep track of # of queries
							if($naccessdenied > $limblankpage){
								print "# Teoma has banned us (page results: $cur). Quitting....\n";
								exit;
							}

							sleep(60+$naccessdenied * 60);
						}
						else{
							$trydifferent=1;
						}
					}
					else{
						# We did get search results

						# If all were collisions, count the number of collisions to exit on threshold.
						if($collisions >= $matches){
							#print "#### DEBUG: GOT ALL COLLISIONS ####\n";
							$ncollisions++;

							if($ncollisions >= 10){
								$trydifferent=1;
							} 

							$cur++;
						}
						else{
							$difmatch=$totalmatches-$matches;
							# Not all results were collisions. So get the next search results.
							$cur++;
							$ncollisions=0;
						}

						# If last search didn't have "Next results", we have no other option than changing the query stringg
						if($hasnext == 0){
							#print "#### DEBUG: NO NEXT RESULTS ####\n";
							$trydifferent=1;
						}
					}  # if/else on whether there were matches
				}  # End of $? (result of wget)

				# XXX Teoma will block us if we query too fast. Still need to tune this to fly low enough
				sleep($cur);
			}while($? == 0 && $trydifferent == 0);

			if($#remsites < ($MAXREMOVEDSITES-1)){
				# If list is not complete, attempt to complete it
				GenDomListLevel($pdomain, \@remsites, $czone, 1, $MAXREMOVEDSITES);
			}
		} # for that adds new removed sites;
}



sub GetDomainsFromDuckduckgo{
#		GetDomainsFromCrawler(\@domain, $czone, $level, $maxresults);	
		my($pdomain, $pzone, $plevel, $pmaxresults) = @_;
		my $czone=$pzone;

		$MAXREMOVEDSITES=25;

# Now set from the config file
#		$MAXTIMEPERIOD=10;

#		$maxtime=$MAXTIMEPERIOD;

#		if($maxtime < 5){
#			$maxtime=5;
#		}

		$MAXCNNORESULTS=125;
		$MINCNNORESULTS= 10;

		$limcnnoresults= $MAXCNNORESULTS/($plevel+1);

		if($limcnnoresults < $MINCNNORESULTS){
			$limcnnoresults= $MINCNNORESULTS;
		}

		$MAXCCOLLISIONS=150;
		$MINCCOLISSIONS= 10;
		$limccollisions= $MAXCCOLLISIONS/($plevel+1);

		if($limccollisions < $MINCCOLISSIONS){
			$limccollisions= $MINCCOLISSIONS;
		}


		# This triggers a change in search string after cumulative number of pages that resulted
		# in 100% collisions
		$cphaseonesubone= $limccollisions/3;
		if($cphaseonesubone<4){
			$cphaseonesubone=4;
		}

		# This triggers changes in subphases that should change faster
		$csubphase= $limccollisions/4;
		if($csubphase < 4){
			$csubphase=4;
		}

		$MAXBLANKPAGE= 25;
		$MINBLANKPAGE= 5;

		$limblankpage= $MAXBLANKPAGE/($plevel+1);

		if($limblankpage < $MINBLANKPAGE){
			$limblankpage= $MINBLANKPAGE;
		}


		$maxresults= $pmaxresults;

		$cur= 1;
		$nresults=0;
		$fpzeromatch=0;
		$nzeromatch=0;
		$czeromatch=0;
		$ncollisions=0;
		$ccollisions=0;
		$cnnoresults=0;
		$continue=1;
		$phase=0;
		$subphase=0;
		$heur="";
		$sitesstr="";
		$ftstr="";
		$remsites="";
		$totalqueries=0;
		$naccessdenied=0;


		do{   # Iteration for each page
			$accessdenied=0;
			$matches=0;
			$totalmatches=0;
			$collisions=0;
			$hasnext=0;

			# We assume there were no results. If there are, we reset $noresults to 0
			$noresults=1;

			# --user-agent "Mozilla/4.0"  works fine most of the time. curl || wget original headers cause trouble.
			$useragent=GetRandomUserAgent();


			$sitesstr=~ s/\+/ /g;
			$sitesstr=~ s/%3A/:/g;
			$ftstr=~ s/\+/ /g;
			$ftstr=~ s/%3A/:/g;
			$heur=~ s/\+/ /g;

			if($cur == 1){
				$nextparams="";
				$numres=30;
			}

# ?q=site%3A$czone$sitesstr$ftstr$heur"`;
			if($wwwclient eq "curl"){
				if($cur == 1){
					if($wwwclient eq "curl"){
#print "#COMANDO: \"$wwwclient $wwwclient_sopt $wwwclient_tout $wwwclient_agnt\"$useragent\" -d q=\"site:$czone$sitesstr$ftstr$heur\" -d b=\"\" \"https://duckduckgo.com/html/\"\n";

						@revname=`$wwwclient $wwwclient_sopt $wwwclient_tout $wwwclient_agnt"$useragent" -d q="site:$czone$sitesstr$ftstr$heur" -d b="" "https://duckduckgo.com/html/"`;
					}
				}
				else{
					@revname=`$wwwclient $wwwclient_sopt $wwwclient_tout $wwwclient_agnt"$useragent" -d q="site%3A$czone$sitesstr$ftstr$heur" -d s=$numres  -d nextParams="$nextparams" -d v=l -d o=json -d dc=$cur  -d api="/d.js" -d kl=us-en "https://duckduckgo.com/html/"`;
				}
			}

#curl -o ipv661.html -d q="ipv6" -d s=30  -d nextParams="" -d v=l -d o=json -d dc=61  -d api="/d.js" -d kl=us-en https://duckduckgo.com/html/


			$totalqueries++;
				
			if($? == 0){
				foreach $line (@revname){

#print "LINE: $line\n";
					if($line=~ m/<div id="accessDenied">/){
						$accessdenied=1;
					}

					if($line=~ m/<input type="submit"/){
#print "#DEBUG: INPUTMATCHSUBMIT\n";

					}

$REGEX="<input type=\"submit\" class='btn btn--alt' value=\"Next\" />";

					if($line=~ m/$REGEX/){
#print "#DEBUG: MATCHNEXT\n";
						$hasnext=1;
					}

					if($hasnext == 1){
#print "#DEBUG: Zona de variables\n";
						if($line=~ m/<input\s+type="hidden"\s+name="s"\s+value="(\d+)"\s*\/>/){
#print "#MATCHIE S\n";
							$numres= $1;
						}
						elsif($line=~ m/<input\s+type="hidden"\s+name="nextParams"\s+value="([^"]*)"\s*\/>/){
							$nextparams=$1;
#print "#MATCHIE nextparams\n";
						}
						elsif($line=~ m/<input\s+type="hidden"\s+name="dc"\s+value="\d+"\s*\/>/){
							$cur=$1;
#print "#MATCHIE dc\n";
						}
					}

					if($line=~ m/<a\s+rel="nofollow"\s+class="result__a"\s+href="https?:\/\/([^\/"]+)/){
						$noresults=0;
						$totalmatches++;
						$grabbeddomain= $1;
#print "MATCHED: $grabbeddomain ###############################################\n";
						if($grabbeddomain !~ m/$czone(:\d+)?/){
							#print "##### DEBUG: Discarded $grabbeddomain\n";
							next;
						}

						$matches++;
						$nzeromatch=0;
						$czeromatch=0;
#print "# DEBUG: Lei dominio $grabbeddomain\n";

						for($i=0; $i<= $#$pdomain; $i++){
#print "# Comparing $pdomain->[$i] ($i) against $grabbeddomain\n";
							if($pdomain->[$i] eq $grabbeddomain){
#print "# Was dupplicate\n";
								$collisions++;
								last;
							}
						}

						if($i > $#$pdomain){
#print "# Added\n";
							if($maxresults < 0 || ($maxresults >= 0 &&  $#$pdomain < $maxresults)){
								$pdomain->[$#$pdomain + 1]=$grabbeddomain;
								print "$grabbeddomain\n";
								$nresults++;
							}
							else{
								$continue=0;
								last;
							}
						}

						if($maxresults >= 0 && $#$pdomain >=$maxresults){
							$continue=0;
							last;
						} 
					} # URL (search result)
				}

#if($noresults == 1){
#print "Didnt' match next. This is the document\n";
#foreach $line (@revname){
#print "#LINE: $line\n";
#}
#}

				#  There are a number of posible results:
				#
				# 1) We didn't get any results:
				#     a) because search engine banned us (condition: no matches, but no "no results" page)
				#        ACTION: Wait for some time, and retry with same query string and same "results page"
				#     b) Because the query string didn't yield results (condition: "no results" page)
				#        ACTION: First couple of times, wait a bit, and retry. Otherwise, change the query string. And
				#                if we hit the threshold of "no results", quit.
				# 2) We got results:
				#     a) Some were new results
				#        ACTION: Keep searching
				#     b) All were collisions
				#        ACTION: If we hit threshold, change query string. Otherwise, keep searching

				$trydifferent=0;

				# totalmatches == 0 is the same as noresults=1
				if($totalmatches == 0){
					if($accessdenied == 1){
						$naccessdenied++;

						if($naccessdenied > $limblankpage){
							print "# DuckDuckGo has banned us (totalqueries: $totalqueries). Quitting....\n";
							exit;
						}

						sleep(60+$naccessdenied * 60);
					}
					else{

						#print "#### DEBUG: GOT NO RESULTS ####\n";
						if($nzeromatch == 0){
							$fpzeromatch=$cur;  # Record "when" this happened
						}

						$nzeromatch++;

						# The continue = 0 condition was set above, anyway
						if($cnnoresults > $limcnnoresults){
							# If we get "no results" after 25 pages we quit
							# print "#### Bing showed no results for our query, after retries. Quitting.... ####\n";
							$continue=0;
						}
						else{
							# Before we were trying the same query. But it does not really make sense.
							$trydifferent=1;
						}
					}
				}
				else{
					# We did get search results

					# If all were collisions, count the number of collisions to exit on threshold.
					if($collisions >= $matches){
						#print "#### DEBUG: GOT ALL COLLISIONS ####\n";
						$ncollisions++;
						$ccollisions++;  # Cumulative colissions even across changes in search string

						if( ($phase >= 2 && $ccollisions > $limccollisions) || ($plevel > 0 && $phase > 0 && $ccollisions > $limccollisions)){
							# If we get nothing after 100 pages (across change of query strings), and we are in 
							# phase 2 or more, we quit
							# print "#### Couldn't get any new results. Quitting.... ####\n";
							$continue=0;
						}

						if($phase == 0){
							if($subphase == 0){
								if($ncollisions > $cphaseonesubone){
									$trydifferent=1;
								}
							}
							else{
								if($ncollisions > $csubphase){
									$trydifferent=1;
								}								
							}
						}
						elsif($phase == 1){
							if($ncollisions > $cphaseonesubone){
								$trydifferent=1;
							}
						}
						elsif($phase == 2){
							if($ncollisions > $csubphase){
								$trydifferent=1;
							}
						}
						else{
							if($ncollisions > $csubphase){
								$trydifferent=1;
							}
						}

						$cur++;
					}
					else{
						$difmatch=$totalmatches-$matches;
						# Not all results were collisions. So get the next search results.
						$cur++;
						$ncollisions=0;
						$ccollisions=0;
						#print "### GOT NEW DATA ####\n"
					}

					# If last search didn't have "Next results", we have no other option than changing the query stringg
					if($hasnext == 0){
						#print "#### DEBUG: NO NEXT RESULTS ####\n";
						$trydifferent=1;
					}

				}  # if/else on whether there were matches

				if($trydifferent == 1){
					# subfase0: no pongo nada extra
					# subfase1: tipos de archivos
					# subfase2: elimino sites
					# subfase3: elimino sites sin www;
					# subfase4: tipos de archivo + elimino sites	
					# subfase5: tipos de archivo ´+ sites sin www

					if($subphase == 0){
						$subphase=1;
						$ftindex=0;
						$ftstr="+filetype%3A$filetypesstr[$ftindex]";
						$ncollisions=0;
						$cur=1; # Since we change the search keys, we retrieve the first page of results
					}
					elsif($subphase == 1){
						if($ftindex >= $#filetypesstr){
							$ftstr="";
							$ftindex=0;
							$subphase=2;
							undef @remsites;
							$sitesstr=GetRemSitesString($pdomain, $czone, 0);
						}
						else{
							$ftindex++;
							$ftstr="+filetype%3A$filetypesstr[$ftindex]";
						}

						$ncollisions=0;
						$cur=1; # Since we change the search keys, we retrieve the first page of results
					}
					elsif($subphase == 2){
						$subphase=3;
						undef @remsites;
						$sitesstr=GetRemSitesString($pdomain, $czone, 1);
						$ncollisions=0;
						$cur=1; # Since we change the search keys, we retrieve the first page of results
					}
					elsif($subphase == 3){
						$subphase=4;
						undef @remsites;
						$sitesstr=GetRemSitesString($pdomain, $czone, 0);
						$ftindex=0;
						$ftstr="+filetype%3A$filetypesstr[$ftindex]";
						$ncollisions=0;
						$cur=1; # Since we change the search keys, we retrieve the first page of results
					}
					elsif($subphase == 4){
						if($ftindex >= $#filetypesstr){
							$subphase=5;
							$ftindex=0;
							$ftstr="+filetype%3A$filetypesstr[$ftindex]";
							undef @remsites;
							$sitesstr=GetRemSitesString($pdomain, $czone, 1);
						}
						else{
							$ftindex++;
							$ftstr="+filetype%3A$filetypesstr[$ftindex]";
						}

						$ncollisions=0;
						$cur=1; # Since we change the search keys, we retrieve the first page of results
					}
					elsif($subphase ==5){
						if($ftindex < $#filetypesstr){
							$ftindex++;
							$ftstr="+filetype%3A$filetypesstr[$ftindex]";
							$ncollisions=0;
							$cur=1; # Since we change the search keys, we retrieve the first page of results
						}
						else{
							$ftstr="";
							$ftindex=0;
							$sitesstr="";
							$subphase=0;

							if($phase == 0){
								# print "#### Phase 1  ####\n";
								$phase=1;
								$letterkey="a";
								$heur="+$letterkey";
								$ncollisions=0;
								$cur=1; # Since we change the search keys, we retrieve the first page of results
							}
							elsif($phase == 1){
								# In phase 1 we search for the domain name and one letter
								if($letterkey eq "z"){
									$letterkey="a";
									$numkey="0";
									$heur="+$numkey";
									$ncollisions=0;
									$phase=2;
									$cur=1; # Since we change the search keys, we retrieve the first page of results
									# print "#### Phase 2  ####\n";
								}
								else{
									$letterkey++;
									$ncollisions=0;
									$heur="+$letterkey";
									$cur=1; # Since we change the search keys, we retrieve the first page of results
								}
							}
							elsif($phase == 2){
								# In phase 2 we search for the domain name and a number

								if($numkey >= 10001){
									$letterkey="a";
									$numkey=0;
									$heur="+$letterkey+$numkey";
									$ncollisions=0;
									$phase=3;
									$cur=1; # Since we change the search keys, we retrieve the first page of results
									# print "#### Phase 3  ####\n";
								}
								else{
									$numkey++;
									$ncollisions=0;
									$cur=1; # Since we change the search keys, we retrieve the first page of results
									$heur="+$numkey";
								}
							}
							else{
								# In phase #3, we search for the domainname, plus a letter and a number
								if($numkey > 10000){
									if($letterkey eq "z"){
										# print "#### Tried my best. Quitting.... ####\n";
										$continue=0;
									}
									else{
										$letterkey++;
										$numkey=0;
										$cur=1; # Since we change the search keys, we retrieve the first page of results
										$heur="+$letterkey+$numkey";
										$ncollisions=0;
									}
								}
								else{
									$numkey++;
									$ncollisions=0;
									$cur=1; # Since we change the search keys, we retrieve the first page of results
									$heur="+$letterkey+$numkey";
								}
							} #end oh phase 
						} # end of else in subphase 5

					} #end of subphase

				}  # End of "trydifferent"
			}  # End of $? (result of wget)


			# XXX Teoma will block us if we query too fast. Still need to tune this to fly low enough
			sleep($cur);
		}while($? == 0 && $continue == 1);


		if($maxresults >= 0 && $#$pdomain >= $maxresults){
			return;
		}


		# We will now exclude next level domains

		undef @remsites;

		GenDomListLevel($pdomain, \@remsites, $czone, 1, $MAXREMOVEDSITES);

		$continue=1;

		for($jjj= 0; $jjj <= $#remsites && $continue == 1; $jjj++){
			$cur= 1;

			$sitesstr= GenRemListStr(\@remsites, $jjj);
			$trydifferent=0;

			do{   # Iteration for each page
				$matches=0;
				$totalmatches=0;
				$collisions=0;
				$hasnext=0;
				$noresults=1;
				$accessdenied=0;

				$useragent=GetRandomUserAgent();

				$sitesstr=~ s/\+/ /g;
				$sitesstr=~ s/%3A/:/g;
				$ftstr=~ s/\+/ /g;
				$ftstr=~ s/%3A/:/g;
				$heur=~ s/\+/ /g;

				if($cur == 1){
					$nextparams="";
					$numres=30;
				}

				# ?q=site%3A$czone$sitesstr$ftstr$heur"`;
				if($wwwclient eq "curl"){
					@revname=`$wwwclient $wwwclient_sopt $wwwclient_tout $wwwclient_agnt"$useragent" -d q="site%3A$czone$sitesstr$ftstr$heur" -d s=$numres  -d nextParams="$nextparams" -d v=l -d o=json -d dc=$cur  -d api="/d.js" -d kl=us-en "https://duckduckgo.com/html/"`;
				}

				$totalqueries++;

				if($? == 0){
					foreach $line (@revname){
						if($continue == 0){
							last;
						}

						if($line=~ m/<div id="accessDenied">/){
							$accessdenied=1;
						}

						if($line=~ m/<input type="submit" class='btn btn--alt' value="Next" \/>/){
							$hasnext=1;
						}

						if($hasnext == 1){
							if($line=~ m/<input\s+type="hidden"\s+name="s"\s+value="(\d+)"\s*\/>/){
								$numres= $1;
							}
							elsif($line=~ m/<input\s+type="hidden"\s+name="nextParams"\s+value="([^"]*)"\s*\/>/){
								$nextparams=$1;
							}
							elsif($line=~ m/<input\s+type="hidden"\s+name="dc"\s+value="\d+"\s*\/>/){
								$cur=$1;
							}
						}

						if($line=~ m/<a\s+rel="nofollow"\s+class="result__a"\s+href="https?:\/\/([^\/"])+/){
#print "# DEBUG: MATCHIE RESULTADO: $1\n";
							$noresults=0;;
							$totalmatches++;
							$grabbeddomain= $1;

							if($grabbeddomain !~ m/$czone(:\d+)?/){
								next;
							}

							$matches++;
							$nzeromatch=0;

							for($k=0; $k<= $#$pdomain; $k++){
								if($pdomain->[$k] eq $grabbeddomain){
									$collisions++;
									last;
								}
							}


							if($k > $#$pdomain){
								if($maxresults < 0 || ($maxresults >= 0 &&  $#$pdomain < $maxresults)){
									$pdomain->[$#$pdomain + 1]=$grabbeddomain;
									print "$grabbeddomain\n";
									$nresults++;
								}
								else{
									$continue=0;
								}
							}

							if($maxresults >= 0 && $#$pdomain>=$maxresults){
								$continue=0;
							} 
						} # while() that reads lines
					}  # foreach $line

					$trydifferent=0;

					if($totalmatches == 0){
						if($accessdenied == 1){
							$naccessdenied++;

							# XXX: Should keep track of # of queries
							if($naccessdenied > $limblankpage){
								print "# DuckDuckGo has banned us (page results: $totalqueries). Quitting....\n";
								exit;
							}

							sleep(60+$naccessdenied * 60);
						}
						else{
							$trydifferent=1;
						}
					}
					else{
						# We did get search results

						# If all were collisions, count the number of collisions to exit on threshold.
						if($collisions >= $matches){
							#print "#### DEBUG: GOT ALL COLLISIONS ####\n";
							$ncollisions++;

							if($ncollisions >= 10){
								$trydifferent=1;
							} 

							$cur++;
						}
						else{
							$difmatch=$totalmatches-$matches;
							# Not all results were collisions. So get the next search results.
							$cur++;
							$ncollisions=0;
						}

						# If last search didn't have "Next results", we have no other option than changing the query stringg
						if($hasnext == 0){
							#print "#### DEBUG: NO NEXT RESULTS ####\n";
							$trydifferent=1;
						}
					}  # if/else on whether there were matches
				}  # End of $? (result of wget)

				# XXX Teoma will block us if we query too fast. Still need to tune this to fly low enough
				sleep($cur);
			}while($? == 0 && $trydifferent == 0);

			if($#remsites < ($MAXREMOVEDSITES-1)){
				# If list is not complete, attempt to complete it
				GenDomListLevel($pdomain, \@remsites, $czone, 1, $MAXREMOVEDSITES);
			}
		} # for that adds new removed sites;
}


# Function GetORG()
#
# Obtains the organization corresponding to an autonomous system number (ASN)
#
sub GetORG{
	$querystring="host -t TXT AS".$_[0].".asn.cymru.com";
	@asinfo= `$querystring`;

	if($asinfo[0] =~ m/\"*.\|.*\|.*\|.*\|\s*(.*)\"/){
		return($1);
	}
	else{
		return("");
	}
}


if($#ARGV < 0){
	print "$SI6_TOOLKIT\n";
	print "$SCRIPT6\n";
	print "$USAGE\n";
}
elsif($ARGV[0] eq "get-a"){
	if($#ARGV == 0){
		while($domain=<STDIN>){
			$aaaa_f=0;
			chomp($domain);

			# If the line cotains a comment at the end of the line, remove it
			if($domain =~ m/^\s*([-a-zA-Z0-9_.]+)/){
				$domcmd= $1;
			}
			else{
				next;
			}

			while(1){
				$answer_f=0;
				$command= "dig $domcmd a 2>&1";
				$pid = open3(\*CHLD_IN, \*CHLD_OUT, \*CHLD_ERR, $command, );
				close(CHLD_IN);
				close(CHLD_ERR);

				while($line=<CHLD_OUT>){
					if($answer_f == 0){
						if($line =~ m/^dig: /){
							print "# Error for $domcmd: $line";
							last;
						}

						# Detect the "ANSWER" portion of the dig command
						if($line =~ m/;; ANSWER SECTION:/){
							$answer_f=1;
						}

						next; 
					}
					elsif($answer_f == 1){
						# If we are in the answer portion, look for AAAA records
						if($line =~ m/^\s*$/){
							last;
						}

						elsif($line =~ m/(\S+)\s+\d+\s+IN\s+A\s+((?:\d{1,3}\.){3}\d{1,3})/){
							if($aaaa_f == 0){
								print "# $domcmd ($1)\n";
							}

							print "$2\n";      # AAAA record
							$aaaa_f = 1;
						}
					}
				}
			 

				waitpid( $pid, 0 );


				# If we found AAAA records, try the next domain
				if($aaaa_f == 1){
					last;
				}

				# If we didn't find any AAAA records, and the domain name does not start with "www",
				# add the "www" suffix and see if there are any AAAA records.
				if($domcmd =~ m/^www\.[-a-zA-Z_0-9.]+/){
					last;
				}
				else{
					$domcmd= "www.$domcmd";
				}
			}


		}
	}
	else{
			# Get AAAA for a domain name given as a parameter
			$domain= $ARGV[1];
			$aaaa_f=0;
			chomp($domain);

			# If the line cotains a comment at the end of the line, remove it
			if($domain =~ m/^\s*([-a-zA-Z0-9_.]+)/){
				$domcmd= $1;
			}
			else{
				next;
			}

			while(1){
				$command= "dig $domcmd a 2>&1";
				$pid = open3(\*CHLD_IN, \*CHLD_OUT, \*CHLD_ERR, $command, );
				close(CHLD_IN);
				close(CHLD_ERR);

				$answer_f=0;

				while($line=<CHLD_OUT>){
					if($answer_f == 0){
						if($line =~ m/^dig: /){
							print "# Error for $domcmd: $line";
							last;
						}

					# Detect the "ANSWER" portion of the dig command
						if($line =~ m/;; ANSWER SECTION:/){
							$answer_f=1;
						}

						next; 
					}
					elsif($answer_f == 1){
						# If we are in the answer portion, look for AAAA records
						if($line =~ m/^\s*$/){
							last;
						}

						elsif($line =~ m/(\S+)\s+\d+\s+IN\s+A\s+((?:\d{1,3}\.){3}\d{1,3})/){
							if($aaaa_f == 0){
								print "# $domcmd ($1)\n";
							}

							print "$2\n";      # AAAA record
							$aaaa_f = 1;
						}
	#					else{
	#						last;
	#					}
					}
				}

				waitpid( $pid, 0 );
			 

				# If we found AAAA records, try the next domain
				if($aaaa_f == 1){
					last;
				}

				# If we didn't find any AAAA records, and the domain name does not start with "www",
				# add the "www" suffix and see if there are any AAAA records.
				if($domcmd =~ m/^www\.[-a-zA-Z_0-9.]+/){
					last;
				}
				else{
					$domcmd= "www.$domcmd";
				}
			}

	}
}
elsif($ARGV[0] eq "get-aaaa"){
	if($#ARGV == 0){
		while($domain=<STDIN>){
			$aaaa_f=0;
			chomp($domain);

			# If the line cotains a comment at the end of the line, remove it
			if($domain =~ m/^\s*([-a-zA-Z0-9_.]+)/){
				$domcmd= $1;
			}
			else{
				next;
			}

			while(1){
				undef @dig;

				@dig=`dig $domcmd aaaa 2>&1`;
				$answer_f=0;

				foreach $line (@dig){
					if($answer_f == 0){
						if($line =~ m/^dig: /){
							print "# Error for $domcmd: $line";
							last;
						}

						# Detect the "ANSWER" portion of the dig command
						if($line =~ m/;; ANSWER SECTION:/){
							$answer_f=1;
						}

						next; 
					}
					elsif($answer_f == 1){
						# If we are in the answer portion, look for AAAA records
						if($line =~ m/^\s*$/){
							last;
						}

						elsif($line =~ m/(\S+)\s+\d+\s+IN\s+AAAA\s+([a-fA-F0-9:]+)/){
							if($aaaa_f == 0){
								print "# $domcmd ($1)\n";
							}

							print "$2\n";      # AAAA record
							$aaaa_f = 1;
						}
	#					else{
	#						last;
	#					}
					}
				}
			 

				# If we found AAAA records, try the next domain
				if($aaaa_f == 1){
					last;
				}

				# If we didn't find any AAAA records, and the domain name does not start with "www",
				# add the "www" suffix and see if there are any AAAA records.
				if($domcmd =~ m/^www\.[-a-zA-Z_0-9.]+/){
					last;
				}
				else{
					$domcmd= "www.$domcmd";
				}
			}
		}
	}
	else{
			# Get AAAA for a domain name given as a parameter
			$domain= $ARGV[1];
			$aaaa_f=0;
			chomp($domain);

			# If the line cotains a comment at the end of the line, remove it
			if($domain =~ m/^\s*([-a-zA-Z0-9_.]+)/){
				$domcmd= $1;
			}
			else{
				next;
			}

			while(1){
				undef @dig;
				@dig=`dig $domcmd aaaa 2>&1`;
				$answer_f=0;

				foreach $line (@dig){
					if($answer_f == 0){
						if($line =~ m/^dig: /){
							print "# Error for $domcmd: $line";
							last;
						}

					# Detect the "ANSWER" portion of the dig command
						if($line =~ m/;; ANSWER SECTION:/){
							$answer_f=1;
						}

						next; 
					}
					elsif($answer_f == 1){
						# If we are in the answer portion, look for AAAA records
						if($line =~ m/^\s*$/){
							last;
						}

						elsif($line =~ m/(\S+)\s+\d+\s+IN\s+AAAA\s+([a-fA-F0-9:]+)/){
							if($aaaa_f == 0){
								print "# $domcmd ($1)\n";
							}

							print "$2\n";      # AAAA record
							$aaaa_f = 1;
						}
	#					else{
	#						last;
	#					}
					}
				}
			 

				# If we found AAAA records, try the next domain
				if($aaaa_f == 1){
					last;
				}

				# If we didn't find any AAAA records, and the domain name does not start with "www",
				# add the "www" suffix and see if there are any AAAA records.
				if($domcmd =~ m/^www\.[-a-zA-Z_0-9.]+/){
					last;
				}
				else{
					$domcmd= "www.$domcmd";
				}
			}

	}
}
elsif($ARGV[0] eq "get-mx"){
	if($#ARGV == 0){
		while($domain=<STDIN>){
				# This variable set to 1 indicates that we already found the MX records
			$mx_f=0;

			chomp($domain);

			# If the line cotains a comment at the end of the line, remove it
			if($domain =~ m/^\s*([-a-zA-Z0-9_.]+)/){
				$domcmd= $1;
			}
			else{
				next;
			}

			# If the domain contains a "www" prefix, remove it.
			if($domcmd =~ m/^www\.(\S+)/){
				$domcmd = $1;
			}

			@dig=`dig $domcmd mx 2>&1`;

			$answer_f=0;
			foreach $line (@dig){
				# Detect the "ANSWER" portion of the dig command
				if($answer_f == 0){
					if($line =~ m/^dig: /){
						print "# Error for $domcmd: $line";
						last;
					}

					if($line =~ m/\;\; ANSWER SECTION:/){
						$answer_f=1; # Found the ANSWER section
					}

					next;  
				}
				elsif($answer_f == 1){
					# If we are in the answer portion, look for MX records
					if($line =~ m/^\s*$/){
						# Empty line marks end of ANSWER section
						last;
					}

					elsif($line =~ m/(\S+)\s+\d+\s+IN\s+MX\s+\d+\s+(\S+)/){
						# FIrst 
						if($mx_f == 0){
							print "# $domcmd ($1)\n";
						}

						printf "$2\n";      # MX record
						$mx_f = 1;
					}
				}
			}
		}
	}
	else{
			$domain= $ARGV[1];
				# This variable set to 1 indicates that we already found the MX records
			$mx_f=0;

			chomp($domain);

			# If the line cotains a comment at the end of the line, remove it
			if($domain =~ m/^\s*([-a-zA-Z0-9_.]+)/){
				$domcmd= $1;
			}
			else{
				next;
			}

			# If the domain contains a "www" prefix, remove it.
			if($domcmd =~ m/^www\.(\S+)/){
				$domcmd = $1;
			}

			@dig=`dig $domcmd mx 2>&1`;

			$answer_f=0;
			foreach $line (@dig){
				# Detect the "ANSWER" portion of the dig command
				if($answer_f == 0){
					if($line =~ m/^dig: /){
						print "# Error for $domcmd: $line";
						last;
					}

					if($line =~ m/\;\; ANSWER SECTION:/){
						$answer_f=1; # Found the ANSWER section
					}

					next;  
				}
				elsif($answer_f == 1){
					# If we are in the answer portion, look for MX records
					if($line =~ m/^\s*$/){
						# Empty line marks end of ANSWER section
						last;
					}

					elsif($line =~ m/(\S+)\s+\d+\s+IN\s+MX\s+\d+\s+(\S+)/){
						# FIrst 
						if($mx_f == 0){
							print "# $domcmd ($1)\n";
						}

						printf "$2\n";      # MX record
						$mx_f = 1;
					}
				}
			}

	}
}
elsif($ARGV[0] eq "get-ptr"){
	if($#ARGV == 0){
		while($domain=<STDIN>){
				# This variable set to 1 indicates that we already found the MX records
			$mx_f=0;

			chomp($domain);

			# If the line cotains a comment at the end of the line, remove it
			if($domain =~ m/^\s*([0-9a-zA-Z.:]+)/){
				$domcmd= $1;
			}
			else{
				next;
			}

			@dig=`dig -x $domcmd 2>&1`;

			$answer_f=0;
			foreach $line (@dig){
				# Detect the "ANSWER" portion of the dig command
				if($answer_f == 0){
					if($line =~ m/^dig: /){
						print "# Error for $domcmd: $line";
						last;
					}

					if($line =~ m/\;\; ANSWER SECTION:/){
						$answer_f=1; # Found the ANSWER section
					}

					next;  
				}
				elsif($answer_f == 1){
					# If we are in the answer portion, look for MX records
					if($line =~ m/^\s*$/){
						# Empty line marks end of ANSWER section
						last;
					}

					elsif($line =~ m/(\S+)\s+\d+\s+IN\s+PTR\s+(\S+)/){
						# FIrst 
						if($mx_f == 0){
							print "# $domcmd ($1)\n";
						}

						print "$2\n";      # MX record
						$mx_f = 1;
					}
				}
			}
		}
	}
	else{
			$domain= $ARGV[1];
				# This variable set to 1 indicates that we already found the MX records
			$mx_f=0;

			chomp($domain);

			# If the line cotains a comment at the end of the line, remove it
			if($domain =~ m/^\s*([0-9a-zA-Z.:]+)/){
				$domcmd= $1;
			}
			else{
				next;
			}


			@dig=`dig -x $domcmd 2>&1`;

			$answer_f=0;
			foreach $line (@dig){
				# Detect the "ANSWER" portion of the dig command
				if($answer_f == 0){
					if($line =~ m/^dig: /){
						print "# Error for $domcmd: $line";
						last;
					}

					if($line =~ m/\;\; ANSWER SECTION:/){
						$answer_f=1; # Found the ANSWER section
					}

					next;  
				}
				elsif($answer_f == 1){
					# If we are in the answer portion, look for MX records
					if($line =~ m/^\s*$/){
						# Empty line marks end of ANSWER section
						last;
					}

					elsif($line =~ m/(\S+)\s+\d+\s+IN\s+PTR\s+(\S+)/){
						# FIrst 
						if($mx_f == 0){
							print "# $domcmd ($1)\n";
						}

						print "$2\n";      # MX record
						$mx_f = 1;
					}
				}
			}

	}
}
elsif($ARGV[0] eq "get-ns"){
	if($#ARGV == 0){
		while($domain=<STDIN>){
			$ns_f=0;
			chomp($domain);

			# If the line cotains a comment at the end of the line, remove it
			# If the line cotains a comment at the end of the line, remove it
			if($domain =~ m/^\s*([-a-zA-Z0-9_.]+)/){
				$domcmd= $1;
			}
			else{
				next;
			}

			# If the domain contains a "www" prefix, remove it.
			if($domcmd =~ m/^www\.(\S+)/){
				$domcmd = $1;
			}

			undef @dig;	
			@dig=`dig $domcmd ns 2>&1`;
			$answer_f=0;

			foreach $line (@dig){
				if($answer_f == 0){
					if($line =~ m/^dig: /){
						print "# Error for $domcmd: $line";
						last;
					}

					# Detect the "ANSWER" portion of the dig command
					if($line =~ m/;; ANSWER SECTION:/){
						$answer_f=1;
					}

					next; 
				}
				elsif($answer_f == 1){
					# If we are in the answer portion, look for AAAA records
					if($line =~ m/^\s*$/){
						# A blank line marks no more ANSWERs/records
						last;
					}

					elsif($line =~ m/(\S+)\s+\d+\s+IN\s+NS\s+(\S+)/){
						if($ns_f == 0){
							# Before the first NS record we print the domain name
							print "# $domcmd ($1)\n";
						}

						printf "$2\n";      # print NS record
						$ns_f = 1;
					}
				}
			}
		}
	}
	else{
			$domain= $ARGV[1];
			$ns_f=0;
			chomp($domain);

			# If the line cotains a comment at the end of the line, remove it
			if($domain =~ m/^\s*([-a-zA-Z0-9_.]+)/){
				$domcmd= $1;
			}
			else{
				exit(0);
			}

			# If domainname is preceded by "www." strip it (we want base domains)
			if($domcmd =~ m/^www\.(\S+)/){
				$domcmd = $1;
			}

			undef @dig;	
			@dig=`dig $domcmd ns 2>&1`;
			$answer_f=0;

			foreach $line (@dig){
				if($answer_f == 0){
					if($line =~ m/^dig: /){
						print "# Error for $domcmd: $line";
						last;
					}

					# Detect the "ANSWER" portion of the dig command
					if($line =~ m/;; ANSWER SECTION:/){
						$answer_f=1;
					}

					next; 
				}
				elsif($answer_f == 1){
					# If we are in the answer portion, look for AAAA records
					if($line =~ m/^\s*$/){
						# A blank line marks no more ANSWERs/records
						last;
					}

					elsif($line =~ m/(\S+)\s+\d+\s+IN\s+NS\s+(\S+)/){
						if($ns_f == 0){
							# Before the first NS record we print the domain name
							print "# $domcmd ($1)\n";
						}

						printf "$2\n";      # print NS record
						$ns_f = 1;
					}
				}
			}

	}
}
elsif($ARGV[0] eq "get-dns6-rev-batch" || $ARGV[0] eq "get-dns6-rev-batch"){
	$resolverfile="/etc/resolv.conf";
	
	if($#ARGV == 0){
		open FILE, "$resolverfile" or die $!;

		while(<FILE>){
			if($_ =~ m/^\s*nameserver\s+([-a-zA-Z0-9.:_]+)/i){
				$dnsresolver=$1;
				last;
			}
		}

		close FILE;

		if(!$dnsresolver){
			$dnsresolver="8.8.8.8";
		}
	}
	else{
		if($ARGV[1] =~ m/([-a-zA-Z0-9.:_]+)/){
			$dnsresolver= $1;
		}
	}

	while($prefix=<STDIN>){
		chomp($prefix);

		# This should be replaced by a proper regex to match an IPv6 address
		if($prefix !~ m/[-a-zA-Z0-9.:_]+\/\d+/){
			next;
		}

		$command= "$dns6revenum $dnsresolver $prefix 2>&1";
		$pid = open3(\*CHLD_IN, \*CHLD_OUT, \*CHLD_ERR, $command, );
		close(CHLD_IN);
		close(CHLD_ERR);

		print "#### PREFIX: $prefix ####\n";

		while($line=<CHLD_OUT>){
#			print "$line"; #32+7?
			if($line=~m/^Found: ((?:[0-9a-fA-F:]){2,39})\s+is\s+([-a-zA-Z0-9.:_]+)\s*$/){
				print "# $1\n";
				print "$2\n";
			}
		}

		waitpid( $pid, 0 );
		print "\n";
	}
}
elsif($ARGV[0] eq "get-sublister-batch" || $ARGV[0] eq "get-sublist3r-batch"){
	if($#ARGV != 1){
		print "Usage: script6 $ARGV[0] SUBLISTERPATH\n";
		exit(1);
	}

	@revname=`whoami`;
	$user= $revname[0];
	chomp($user);
	@revname=`hostname`;
	$hostname= $revname[0];
	chomp($hostname);

	while($domain=<STDIN>){
		chomp($domain);

		# Skip lines with comments
		if($domain =~ m/^\s*#/){
			next;
		}

		$domain=~ s/[^-a-zA-Z0-9._]//g;

		@revname=`date --iso-8601=seconds -u`;
		$datetime=$revname[0];
		$datetime=~ s/[ :+]//g;
		chomp($datetime);
		$filename="$domain-$datetime-$hostname-$user.txt";
		#open FILE, ">$filename" or die $!;
		#close FILE;


		undef @revname;
		@revname=`$ARGV[1] -d $domain -o $filename 2>&1`;

#		print "#### DOMAIN: $domain ####\n\n";
#
#		foreach $line (@revname){
#			print "$line";
#		}

#		print "\n\n";
	}
}
elsif($ARGV[0] eq "get-suffix-for-tld" || $ARGV[0] eq "get-suffixes-for-tld"){
	if($#ARGV > 1){
		print "Syntax: script6 $ARGV[0] TLD\n";
		exit 1;
	}

	ParseConfigFile();

	if($#ARGV == 1){
		$tld= $ARGV[1];
		SuffixForTLD($tld);
	}
	else{
		while($line=<STDIN>){
			if($line=~ m/^\s*$/){
				next;
			}
			if($line=~ m/^\s*#/){
				print "$line";
			}
			elsif($line=~ m/^\s*([^#]+)/){
				$tld=$1;

				SuffixForTLD($tld);
			}
		}
	}
}
elsif($ARGV[0] eq "get-iso2-for-country"){
	if($#ARGV > 1){
		print "Syntax: script6 get-iso2-for-country [COUNTRY]\n";
	}

	ParseConfigFile();

	if($#ARGV == 1){
		$country= $ARGV[1];
		Iso2ForCountry($country);
	}
	else{
		while($line=<STDIN>){
			if($line=~ m/^\s*$/){
				next;
			}
			if($line=~ m/^\s*#/){
				print "$line";
			}
			elsif($line=~ m/^\s*([^#\s]+)/){
				$country=$1;
				Iso2ForCountry($country);
			}
		}
	}
}
elsif($ARGV[0] eq "get-iso3-for-country"){
	if($#ARGV > 1){
		print "Syntax: script6 get-iso3-for-country [COUNTRY]\n";
	}

	ParseConfigFile();

	if($#ARGV == 1){
		$country= $ARGV[1];
		Iso3ForCountry($country);
	}
	else{
		while($line=<STDIN>){
			if($line=~ m/^\s*$/){
				next;
			}
			if($line=~ m/^\s*#/){
				print "$line";
			}
			elsif($line=~ m/^\s*([^#\s]+)/){
				$country=$1;
				Iso3ForCountry($country);
			}
		}
	}
}
elsif($ARGV[0] eq "get-tld-for-country"){
	if($#ARGV > 1){
		print "Syntax: script6 get-tld-for-country [COUNTRY]\n";
	}

	ParseConfigFile();

	if($#ARGV == 1){
		$country= $ARGV[1];
		open FILE, "$country_database" or die $!;

		while(<FILE>){
			if($_ =~ m/"$country"\s*,\s*"[^"]+"\s*,\s*"[^"]+"\s*,\s*"([^"]+)"\s*/i){
				print "$1\n";
				close(FILE);
				exit;
			}
		}

		close(FILE);
	}
	else{
		while($line=<STDIN>){
			if($line=~ m/^\s*$/){
				next;
			}
			if($line=~ m/^\s*#/){
				print "$line";
			}
			elsif($line=~ m/^\s*([^#\s]+)/){
				$tld=$1;


			}
		}
	}
}
elsif($ARGV[0] eq "get-tld-for-continent"){
	if($#ARGV > 1){
		print "Syntax: script6 get-tld-for-continent [CONTINENT]\n";
	}

	ParseConfigFile();

	if($#ARGV == 1){
		$continent= $ARGV[1];
		open FILE, "$country_database" or die $!;

		while(<FILE>){
			if($_ =~ m/\s*"[^"]+"\s*,\s*"[^"]+"\s*,\s*"[^"]+"\s*,\s*"([^"]+)"\s*,\s*"[^"]+"\s*,\s*"[^"]+"\s*,\s*"[^"]+"\s*,\s*"[^"]+"\s*,\s*"[^"]+"\s*,\s*"$continent"\s*/i){
				print "$1\n";
			}
		}

		close(FILE);
	}
	else{
		while($line=<STDIN>){
			if($line=~ m/^\s*$/){
				next;
			}
			if($line=~ m/^\s*#/){
				print "$line";
			}
			elsif($line=~ m/^\s*([^#\s]+)/){
				$tld=$1;


			}
		}
	}
}
elsif($ARGV[0] eq "get-country-for-continent"){
	if($#ARGV > 1){
		print "Syntax: script6 get-country-for-continent [CONTINENT]\n";
	}

	ParseConfigFile();

	if($#ARGV == 1){
		$continent= $ARGV[1];
		CountryForContinent($continent);
	}
	else{
		while($line=<STDIN>){
			if($line=~ m/^\s*$/){
				next;
			}
			if($line=~ m/^\s*#/){
				print "$line";
			}
			elsif($line=~ m/^\s*([^#\s]+)/){
				$continent=$1;
				CountryForContinent($continent);
			}
		}
	}
}
elsif($ARGV[0] eq "get-iso2-for-rir"){
	if($#ARGV > 1){
		print "Syntax: script6 get-iso2-for-rir [RIR]\n";
	}

	ParseConfigFile();

	if($#ARGV == 1){
		$trir= $ARGV[1];
		Iso2ForRIR($trir);
	}
	else{
		while($line=<STDIN>){
			if($line=~ m/^\s*$/){
				next;
			}
			if($line=~ m/^\s*#/){
				print "$line";
			}
			elsif($line=~ m/^\s*([^#\s]+)/){
				$trir=$1;
				Iso2ForRIR($trir);
			}
		}
	}

}
elsif($ARGV[0] eq "get-tld-for-rir"){
	if($#ARGV > 1){
		print "Syntax: script6 get-iso2-for-rir [RIR]\n";
	}

	ParseConfigFile();

	if($#ARGV == 1){
		$trir= $ARGV[1];
		TLDForRIR($trir);
	}
	else{
		while($line=<STDIN>){
			if($line=~ m/^\s*$/){
				next;
			}
			if($line=~ m/^\s*#/){
				print "$line";
			}
			elsif($line=~ m/^\s*([^#\s]+)/){
				$trir=$1;
				TLDForRIR($trir);
			}
		}
	}

}
elsif($ARGV[0] eq "get-rir-database"){
		ParseConfigFile();
		$useragent=GetRandomUserAgent();
		@lines=`$wwwclient $wwwclient_sopt $wwwclient_agnt"$useragent" "https://www.nro.net/about-the-nro/list-of-country-codes-and-rirs-ordered-by-country-code/"`;

		my @domains;
		$start=0;

		if($? == 0){
			foreach $line (@lines){
				$line=~ s/&#8217;/'/;

				if($start == 0 && $line =~ m/\s*<td\s+class="cc-head">Region<\/td>/){
					$start=1;
				}
				elsif($start == 1){
					if(!$country){
						if($line=~/<td\s+class="cc-(?:white|grey)">([^<*]+)[*\s]*<\/td>/){
							$country=$1;
						}
					}
					elsif(!$iso2){
						if($line=~/<td\s+class="cc-(?:white|grey)">([^<*]+)[*\s]*<\/td>/){
							$iso2=$1;
						}
					}
					elsif(!$iso3){
						if($line=~/<td\s+class="cc-(?:white|grey)">([^<*]+)[*\s]*<\/td>/){
							$iso3=$1;
						}
					}
					elsif(!$rir){
						if($line=~/<td\s+class="cc-(?:white|grey)">([^<*]+)[*\s]*<\/td>/){
							$rir=$1;

							if($iso2 eq "GB" || $iso2 eq "gb"){
								$iso2="UK";
							}

							print "$country;$iso2;$iso3;$rir\n";

							undef $country;
							undef $iso2;
							undef $iso3;
							undef $rir;
						}
					}
				}

			} # foreach $line;
		}   # if $? == 0
}
elsif($ARGV[0] eq "get-crt" || $ARGV[0] eq "get-crt.sh" || $ARGV[0] eq "get-cert-transparency" || $ARGV[0] eq "get-cert-transp"){
	if($#ARGV != 1){
		print "usage: script6 $ARGV[0] DOMAIN\n";
		exit(1);
	}

	$czone= $ARGV[1];
	if($czone =~ m/^\s*\.*([-a-zA-Z0-9.]+)/){
		$czone= $1;
	}

	undef @domain;

	ParseConfigFile();
	GetDomainsFromCrt(\@domain, $czone, $maxresults, \*STDOUT);	
}
elsif($ARGV[0] eq "get-crt-batch"){
	if($#ARGV >=1 ){
		$maxresults= $ARGV[1];
		$maxresults--;

		if($maxresults < 0){
			exit 0;
		}
	}
	else{
		$maxresults= -1;
	}

	ParseConfigFile();

	while($czone=<STDIN>){


		undef @domain;

		if($czone=~ m/^\s*#/){
			next;
		}

		if($czone =~ m/^\s*\.*([-a-zA-Z0-9.]+)/){
			$czone= $1;
		}
		else{
			next;
		}

		print "######## DOMAIN: $czone ########\n";


		GetDomainsFromCrt(\@domain, $czone, $maxresults, \*STDOUT);	
	}
}

elsif($ARGV[0] eq "get-crt-batch-files"){

	if($#ARGV >=1 ){
		$maxresults= $ARGV[1];
		$maxresults--;

		if($maxresults < 0){
			exit 0;
		}
	}
	else{
		$maxresults= -1;
	}

	ParseConfigFile();

	@revname=`whoami`;
	$user= $revname[0];
	chomp($user);
	@revname=`hostname`;
	$hostname= $revname[0];
	chomp($hostname);


	while($czone=<STDIN>){
		undef @domain;

		if($czone=~ m/^\s*#/){
			next;
		}

		if($czone =~ m/^\s*\.*([-a-zA-Z0-9.]+)/){
			$czone= $1;
		}
		else{
			next;
		}

		@revname=`date --iso-8601=seconds -u`;
		$datetime=$revname[0];
		$datetime=~ s/[ :+]//g;
		chomp($datetime);
		$filename="$czone-$datetime-$hostname-$user.txt";
		open FILE, ">$filename" or die $!;

		GetDomainsFromCrt(\@domain, $czone, $maxresults, \*FILE);
		close FILE;	
	}

}

elsif($ARGV[0] eq "get-bing-batch" || $ARGV[0] eq "get-bing4-batch" || $ARGV[0] eq "get-bing6-batch"){
	if($#ARGV >=1 ){
		$maxresults= $ARGV[1];
		$maxresults--;

		if($maxresults < 0){
			exit 0;
		}
	}
	else{
		$maxresults= -1;
	}

	if($ARGV[0] eq "get-bing4-batch"){
		$ipprotocol=4;
	}
	elsif($ARGV[0] eq "get-bing6-batch"){
		$ipprotocol=6;
	}
	else{
		$ipprotocol=0;
	}

	ParseConfigFile();

	while(<STDIN>){


		undef @domain;

		if($_ =~ m/^\s*\.*([-a-zA-Z0-9.]+)/){
			$czone= $1;
		}
		else{
			next;
		}

		$czone= join ".", split /\./, $czone;

		print "######## DOMAIN: $czone ########\n";


		GetDomainsFromBing(\@domain, $czone, 0, $maxresults, $ipprotocol);	

		#print "#### IMPORTANT: WILL NOW SEARCH FOR DOMAINS WITHIN SPECIFIC SUBDOMAINS\n";
		undef @remsites;
		undef @levsites;
		$continue=1;

		$domlevel=1;
		GenDomListLevel(\@domain, \@levsites, $czone, $domlevel, -1);


		while($#levsites >= 0 && $continue == 1){
			#print "#### LEVEL: $domlevel, TOTAL: $#levsites\n\n#LIST:\n";

			for($jj= 0; $jj <= $#levsites && $continue == 1; $jj++){
				$newczone= $levsites[$jj];

				GetDomainsFromBing(\@domain, $newczone, $domlevel, $maxresults, $ipprotocol);
				if($maxresults >= 0 && $#domain>=$maxresults){
					$continue=0;
				}
				else{
					$continue=1;
				}
			}

			$domlevel++;
			undef @levsites;
			GenDomListLevel(\@domain, \@levsites, $czone, $domlevel, -1);
		}  # While lev > 0;
	};  # while(<STDIN>)

	exit;
}
elsif($ARGV[0] eq "get-bing-tt"){
	if($#ARGV < 1 || $#ARGV > 2){
		print "Syntax: script6 get-bing DOMAIN [MAXRESULTS]\n";
		exit 1;
	}

#	$tstamp= time;
#	if($#ARGV == 2){
#		$maxresults= $ARGV[2];
#
#		if($maxresults == 0){
#			exit 0;
#		}
#
#		$maxresults--;
#	}
#	else{
#		$maxresults= -1;
#	}
	$maxresults=-1;

	ParseConfigFile();
	undef @domain;

	$_= $ARGV[1];

	if($_ =~ m/^\s*\.*([-a-zA-Z0-9.]+)/){
		$czone= $1;
	}
	else{
		next;
	}

	$czone= join ".", split /\./, $czone;

	print "######## DOMAIN: $czone ########\n";

	GetDomainsFromBingTT(\@domain, $czone, 0, $maxresults);	# bis

	#print "#### IMPORTANT: WILL NOW SEARCH FOR DOMAINS WITHIN SPECIFIC SUBDOMAINS\n";
#	undef @remsites;
#	undef @levsites;
#	$continue=1;
#
#	$domlevel=1;
#	GenDomListLevel(\@domain, \@levsites, $czone, $domlevel, -1);

#	while($#levsites >= 0 && $continue == 1){
		#print "#### LEVEL: $domlevel, TOTAL: $#levsites\n\n#LIST:\n";

#		for($jj= 0; $jj <= $#levsites && $continue == 1; $jj++){
#			if( (time - $tstamp) > 50){
#print "expiro timer general\n";
#				exit(0);
#			}

#			$newczone= $levsites[$jj];

#			GetDomainsFromBingbis(\@domain, $newczone, $domlevel, $maxresults);
#			if($maxresults >= 0 && $#domain>=$maxresults){
#				$continue=0;
#			}
#			else{
#				$continue=1;
#			}
#		}
#
#		$domlevel++;
#		undef @levsites;
#		GenDomListLevel(\@domain, \@levsites, $czone, $domlevel, -1);
#	}  # While lev > 0;

	exit;
}
elsif($ARGV[0] eq "get-bing" || $ARGV[0] eq "get-bing4" || $ARGV[0] eq "get-bing6"){
	if($#ARGV < 1 || $#ARGV > 2){
		print "Syntax: script6 $ARGV[0] DOMAIN [MAXRESULTS]\n";
		exit 1;
	}

	if($#ARGV == 2){
		$maxresults= $ARGV[2];

		if($maxresults == 0){
			exit 0;
		}

		$maxresults--;
	}
	else{
		$maxresults= -1;
	}

	if($ARGV[0] eq "get-bing4"){
		$ipprotocol=4;
	}
	elsif($ARGV[0] eq "get-bing6"){
		$ipprotocol=6;
	}
	else{
		$ipprotocol=0;
	}

	ParseConfigFile();
	undef @domain;

	$_= $ARGV[1];

	if($_ =~ m/^\s*\.*([-a-zA-Z0-9.]+)/){
		$czone= $1;
	}
	else{
		next;
	}

	$czone= join ".", split /\./, $czone;

	print "######## DOMAIN: $czone ########\n";

	GetDomainsFromBing(\@domain, $czone, 0, $maxresults, $ipprotocol);	

	#print "#### IMPORTANT: WILL NOW SEARCH FOR DOMAINS WITHIN SPECIFIC SUBDOMAINS\n";
	undef @remsites;
	undef @levsites;
	$continue=1;

	$domlevel=1;
	GenDomListLevel(\@domain, \@levsites, $czone, $domlevel, -1);

	while($#levsites >= 0 && $continue == 1){
		#print "#### LEVEL: $domlevel, TOTAL: $#levsites\n\n#LIST:\n";

		for($jj= 0; $jj <= $#levsites && $continue == 1; $jj++){
			$newczone= $levsites[$jj];

			GetDomainsFromBing(\@domain, $newczone, $domlevel, $maxresults, $ipprotocol);
			if($maxresults >= 0 && $#domain>=$maxresults){
				$continue=0;
			}
			else{
				$continue=1;
			}
		}

		$domlevel++;
		undef @levsites;
		GenDomListLevel(\@domain, \@levsites, $czone, $domlevel, -1);
	}  # While lev > 0;

	exit;
}
elsif($ARGV[0] eq "get-bing-by-addr" || $ARGV[0] eq "get-bing4-by-addr" || $ARGV[0] eq "get-bing6-by-addr"){
	if($#ARGV < 1 || $#ARGV > 2){
		print "Syntax: script6 $ARGV[0] ADDRESS [MAXRESULTS]\n";
		exit 1;
	}

	if($#ARGV == 2){
		$maxresults= $ARGV[2];

		if($maxresults == 0){
			exit 0;
		}

		$maxresults--;
	}
	else{
		$maxresults= -1;
	}

	if($ARGV[0] eq "get-bing4"){
		$ipprotocol=4;
	}
	elsif($ARGV[0] eq "get-bing6"){
		$ipprotocol=6;
	}
	else{
		$ipprotocol=0;
	}

	ParseConfigFile();
	undef @domain;

	$czone= $ARGV[1];
	$_= $ARGV[1];


	if($_ =~ m/^(\d{1,3})(?:-(\d{1,3}))?\.(\d{1,3})(?:-(\d{1,3}))?\.(\d{1,3})(?:-(\d{1,3}))?\.(\d{1,3})(?:-(\d{1,3}))?/){
		$a0=$1; $a1=$2; $b0=$3; $b1=$4; $c0=$5; $c1=$6; $d0=$7; $d1=$8;
	}
	else{
		print "Not an IP address\n";
		exit(1);
	}


	if(!defined($2)){
		$a1=$a0;
	}

	if(!defined($4)){
		$b1=$b0;
	}

	if(!defined($6)){
		$c1=$c0;
	}

	if(!defined($8)){
		$d1=$d0;
	}

	if($a0 > $a1 || $b0 > $b1 || $c0 > $c1 || $d0 > $d1){
		print "Bad IP range";
		exit(1);
	}

#	print "a0: $a0; a1: $a1; b0: $b0; b1: $b1; c0: $c0; c1: $c1; d0: $d0: d1: $d1\n";
#	exit(0);

	print "######## ADDRESS: $czone ########\n";

	for($i= $a0; $i<= $a1; $i++){
		for($j= $b0; $j<= $b1; $j++){
			for($k= $c0; $k<= $c1; $k++){
				for($l= $d0; $l<= $d1; $l++){
					GetDomainsFromBingByAddr(\@domain, "$i.$j.$k.$l", $maxresults, $ipprotocol);
				}
			}
		}
	}


	exit;
}
elsif($ARGV[0] eq "get-bing-by-addr-quick" || $ARGV[0] eq "get-bing4-by-addr-quick" || $ARGV[0] eq "get-bing6-by-addr-quick"){
	if($#ARGV < 1 || $#ARGV > 2){
		print "Syntax: script6 $ARGV[0] ADDRESS [MAXRESULTS]\n";
		exit 1;
	}

	if($#ARGV == 2){
		$maxresults= $ARGV[2];

		if($maxresults == 0){
			exit 0;
		}

		$maxresults--;
	}
	else{
		$maxresults= -1;
	}

	if($ARGV[0] eq "get-bing4"){
		$ipprotocol=4;
	}
	elsif($ARGV[0] eq "get-bing6"){
		$ipprotocol=6;
	}
	else{
		$ipprotocol=0;
	}

	ParseConfigFile();
	undef @domain;

	$czone= $ARGV[1];
	$_= $ARGV[1];


	if($_ =~ m/^(\d{1,3})(?:-(\d{1,3}))?\.(\d{1,3})(?:-(\d{1,3}))?\.(\d{1,3})(?:-(\d{1,3}))?\.(\d{1,3})(?:-(\d{1,3}))?/){
		$a0=$1; $a1=$2; $b0=$3; $b1=$4; $c0=$5; $c1=$6; $d0=$7; $d1=$8;
	}
	else{
		print "Not an IP address\n";
		exit(1);
	}


	if(!defined($2)){
		$a1=$a0;
	}

	if(!defined($4)){
		$b1=$b0;
	}

	if(!defined($6)){
		$c1=$c0;
	}

	if(!defined($8)){
		$d1=$d0;
	}

	if($a0 > $a1 || $b0 > $b1 || $c0 > $c1 || $d0 > $d1){
		print "Bad IP range";
		exit(1);
	}

#	print "a0: $a0; a1: $a1; b0: $b0; b1: $b1; c0: $c0; c1: $c1; d0: $d0: d1: $d1\n";
#	exit(0);

	print "######## ADDRESS: $czone ########\n";

	for($i= $a0; $i<= $a1; $i++){
		for($j= $b0; $j<= $b1; $j++){
			for($k= $c0; $k<= $c1; $k++){
				for($l= $d0; $l<= $d1; $l++){
					GetDomainsFromBingByAddrQuick(\@domain, "$i.$j.$k.$l", $maxresults, $ipprotocol);
				}
			}
		}
	}


	exit;
}
elsif($ARGV[0] eq "get-bing-dict" || $ARGV[0] eq "get-bing4-dict" || $ARGV[0] eq "get-bing6-dict"){
	#
	# $ARGV[0]: get-bin-dict
	# $ARGV[1]: domain
	# $ARGV[2]: dictionary
	# $ARGV[3]: maxsites

	if($#ARGV < 2 || $#ARGV > 3){
		print "Syntax: script6 $ARGV[0] DOMAIN DICTIONARY [MAXRESULTS]\n";
		exit 1;
	}

	if($#ARGV == 3){
		$maxresults= $ARGV[3];

		if($maxresults == 0){
			exit 0;
		}

		$maxresults--;
	}
	else{
		$maxresults= -1;
	}

	$dictfile= $ARGV[2];

	if($ARGV[0] eq "get-bing4-dict"){
		$ipprotocol=4;
	}
	elsif($ARGV[0] eq "get-bing6-dict"){
		$ipprotocol=6;
	}
	else{
		$ipprotocol=0;
	}

	ParseConfigFile();
	undef @domain;

	$_= $ARGV[1];

	if($_ =~ m/^\s*\.*([-a-zA-Z0-9._]+)/){
		$czone= $1;
	}
	else{
		print "Invalid DNS zone\n";
		exit(1);
	}

	$czone= join ".", split /\./, $czone;


	print "######## DOMAIN: $czone ########\n";

#	open FILE, "$dictfile" or die $!;
	open my $dictfp, '<', $dictfile or die $!;
	GetDomainsFromBingDict(\@domain, $czone, 0, $maxresults, $dictfp, $ipprotocol, "ANY");	
	close $dictfp;

	#print "#### IMPORTANT: WILL NOW SEARCH FOR DOMAINS WITHIN SPECIFIC SUBDOMAINS\n";
	undef @remsites;
	undef @levsites;
	$continue=1;

	$domlevel=1;
	GenDomListLevel(\@domain, \@levsites, $czone, $domlevel, -1);

	while($#levsites >= 0 && $continue == 1){
		#print "#### LEVEL: $domlevel, TOTAL: $#levsites\n\n#LIST:\n";

		for($jj= 0; $jj <= $#levsites && $continue == 1; $jj++){
			$newczone= $levsites[$jj];

			#	open FILE, "$dictfile" or die $!;
			open my $dictfp, '<', $dictfile or die $!;

			GetDomainsFromBingDict(\@domain, $newczone, $domlevel, $maxresults, $dictfp, $ipprotocol, "ANY");
			if($maxresults >= 0 && $#domain>=$maxresults){
				$continue=0;
			}
			else{
				$continue=1;
			}

			close $dictfp;
		}

		$domlevel++;
		undef @levsites;
		GenDomListLevel(\@domain, \@levsites, $czone, $domlevel, -1);
	}  # While lev > 0;

	exit;
}
elsif($ARGV[0] eq "get-bing-dict-quick" || $ARGV[0] eq "get-bing4-dict-quick" || $ARGV[0] eq "get-bing6-dict-quick"){
	#
	# $ARGV[0]: get-bin-dict
	# $ARGV[1]: domain
	# $ARGV[2]: dictionary
	# $ARGV[3]: maxsites

	if($#ARGV < 2 || $#ARGV > 3){
		print "Syntax: script6 $ARGV[0] DOMAIN DICTIONARY [MAXRESULTS]\n";
		exit 1;
	}

	if($#ARGV == 3){
		$maxresults= $ARGV[3];

		if($maxresults == 0){
			exit 0;
		}

		$maxresults--;
	}
	else{
		$maxresults= -1;
	}

	$dictfile= $ARGV[2];

	if($ARGV[0] eq "get-bing4-dict-quick"){
		$ipprotocol=4;
	}
	elsif($ARGV[0] eq "get-bing6-dict-quick"){
		$ipprotocol=6;
	}
	else{
		$ipprotocol=0;
	}

	ParseConfigFile();
	undef @domain;

	$_= $ARGV[1];

	if($_ =~ m/^\s*\.*([-a-zA-Z0-9._]+)/){
		$czone= $1;
	}
	else{
		print "Invalid DNS zone\n";
		exit(1);
	}

	$czone= join ".", split /\./, $czone;


	print "######## DOMAIN: $czone ########\n";

#	open FILE, "$dictfile" or die $!;
	open my $dictfp, '<', $dictfile or die $!;
	GetDomainsFromBingDict(\@domain, $czone, 0, $maxresults, $dictfp, $ipprotocol, "ANY");	
	close $dictfp;

	exit;
}
elsif($ARGV[0] eq "get-bing-dict-batch" || $ARGV[0] eq "get-bing4-dict-batch" || $ARGV[0] eq "get-bing6-dict-batch"){
	if($#ARGV < 1 ){
		print "usage: script6 $ARGV[0] DICTIONARY [MAXRESULTS]\n";
		exit(1);
	}

	if($#ARGV > 1){
		$maxresults= $ARGV[2];
		$maxresults--;

		if($maxresults < 0){
			exit(0);
		}
	}
	else{
		$maxresults= -1;
	}

	$dictfile= $ARGV[1];


	if($ARGV[0] eq "get-bing4-dict-batch"){
		$ipprotocol=4;
	}
	elsif($ARGV[0] eq "get-bing6-dict-batch"){
		$ipprotocol=6;
	}
	else{
		$ipprotocol=0;
	}

	ParseConfigFile();

	while(<STDIN>){


		undef @domain;

		if($_ =~ m/^\s*\.*([-a-zA-Z0-9.]+)/){
			$czone= $1;
		}
		else{
			next;
		}

		$czone= join ".", split /\./, $czone;

		print "######## DOMAIN: $czone ########\n";


		open my $dictfp, '<', $dictfile or die $!;
		GetDomainsFromBingDict(\@domain, $czone, 0, $maxresults, $dictfp, $ipprotocol, "ANY");	
		close $dictfp;

		#print "#### IMPORTANT: WILL NOW SEARCH FOR DOMAINS WITHIN SPECIFIC SUBDOMAINS\n";
		undef @remsites;
		undef @levsites;
		$continue=1;

		$domlevel=1;
		GenDomListLevel(\@domain, \@levsites, $czone, $domlevel, -1);


		while($#levsites >= 0 && $continue == 1){
			#print "#### LEVEL: $domlevel, TOTAL: $#levsites\n\n#LIST:\n";

			for($jj= 0; $jj <= $#levsites && $continue == 1; $jj++){
				$newczone= $levsites[$jj];

				#	open FILE, "$dictfile" or die $!;
				open my $dictfp, '<', $dictfile or die $!;

				GetDomainsFromBingDict(\@domain, $newczone, $domlevel, $maxresults, $dictfp, $ipprotocol, "ANY");
				if($maxresults >= 0 && $#domain>=$maxresults){
					$continue=0;
				}
				else{
					$continue=1;
				}

				close $dictfp;
			}

			$domlevel++;
			undef @levsites;
			GenDomListLevel(\@domain, \@levsites, $czone, $domlevel, -1);
		}  # While lev > 0;
	};  # while(<STDIN>)

	exit;
}
elsif($ARGV[0] eq "get-bing-dict-quick-batch" || $ARGV[0] eq "get-bing4-dict-quick-batch" || $ARGV[0] eq "get-bing6-dict-quick-batch"){
	if($#ARGV < 1 ){
		print "usage: script6 $ARGV[0] DICTIONARY [MAXRESULTS]\n";
		exit(1);
	}

	if($#ARGV > 1){
		$maxresults= $ARGV[2];
		$maxresults--;

		if($maxresults < 0){
			exit(0);
		}
	}
	else{
		$maxresults= -1;
	}

	$dictfile= $ARGV[1];

	if($ARGV[0] eq "get-bing4-dict-quick-batch"){
		$ipprotocol=4;
	}
	elsif($ARGV[0] eq "get-bing6-dict-quick-batch"){
		$ipprotocol=6;
	}
	else{
		$ipprotocol=0;
	}

	ParseConfigFile();

	while(<STDIN>){


		undef @domain;

		if($_ =~ m/^\s*\.*([-a-zA-Z0-9.]+)/){
			$czone= $1;
		}
		else{
			next;
		}

		$czone= join ".", split /\./, $czone;

		print "######## DOMAIN: $czone ########\n";


		open my $dictfp, '<', $dictfile or die $!;
		GetDomainsFromBingDict(\@domain, $czone, 0, $maxresults, $dictfp, $ipprotocol, "ANY");	
		close $dictfp;
	};  # while(<STDIN>)

	exit;
}

####### PARA ADDRESSES ###############

elsif($ARGV[0] eq "get-bing-addr-dict-quick" || $ARGV[0] eq "get-bing6-addr-dict-quick"){
	#
	# $ARGV[0]: get-bin-dict
	# $ARGV[1]: domain
	# $ARGV[2]: dictionary
	# $ARGV[3]: maxsites

	if($#ARGV < 3 || $#ARGV > 4){
		print "Syntax: script6 $ARGV[0] DOMAIN DICTIONARY IFACE#PREFIX#CHANGE [MAXRESULTS]\n";
		exit 1;
	}

	if($#ARGV == 4){
		$maxresults= $ARGV[4];

		if($maxresults == 0){
			exit 0;
		}

		$maxresults--;
	}
	else{
		$maxresults= -1;
	}

	$dictfile= $ARGV[2];

	$changeinsane= 0;

	@changeaddr= split /#/, $ARGV[3];

	if($#changeaddr < 1 || $#changeaddr > 2){
		print "binding address to be specified as IFACE#PREFIX#CHANGE\n";
		exit(1);
	}
	elsif($#changeaddr == 2){
		if($changeaddr[2] =~ m/insane/){
			$changeinsane=1;
		}
	}

	($chaddr, $chpref) = split /\//, $changeaddr[1];

	if(!$chpref){
		$chpref=64;
	}

	$iface=$changeaddr[0];

	@revname=`uname`;
	$chos=$revname[0];

	if($chos =~ m/linux/i){
		$chos="linux";
	}
	elsif($chos =~ m/bsd/i){
		$chos="bsd";
	}
	else{
		print "Your OS is not supported for this function\n";
	}

#	print "IFACE: $changeaddr[0], ADDR: $chaddr, PREFLEN: $chpref, CHANGEINSANE: $changeinsane, OS: $chos\n";

	$SIG{INT} = \&SigIntHandler;

	$gotaddress=AddAddress($changeaddr[0], $chaddr, $chpref, $chos);
#	print "Randomice: $gotaddress\n";

#	DelAddress($changeaddr[0], $chaddr, $chpref, $chos);


	if($ARGV[0] eq "get-bing6-dict-quick"){
		$ipprotocol=6;
	}
	else{
		# Rather than leaving the protocol unspecified, we force IPv6
		$ipprotocol=6;
	}

	ParseConfigFile();
	undef @domain;

	$_= $ARGV[1];

	if($_ =~ m/^\s*\.*([-a-zA-Z0-9._]+)/){
		$czone= $1;
	}
	else{
		print "Invalid DNS zone\n";
		exit(1);
	}

	$czone= join ".", split /\./, $czone;


	print "######## DOMAIN: $czone ########\n";

#	open FILE, "$dictfile" or die $!;
	open my $dictfp, '<', $dictfile or die $!;
	GetDomainsFromBingDict(\@domain, $czone, 0, $maxresults, $dictfp, $ipprotocol, $gotaddress);
	DelAddress($changeaddr[0], $gotaddress, $chpref, $chos);
	close $dictfp;

	exit;
}
elsif($ARGV[0] eq "get-bing-addr-dict-batch" || $ARGV[0] eq "get-bing6-addr-dict-batch"){
	if($#ARGV < 1 ){
		print "usage: script6 $ARGV[0] DICTIONARY ADDRESS [MAXRESULTS]\n";
		exit(1);
	}

	if($#ARGV > 2){
		$maxresults= $ARGV[3];
		$maxresults--;

		if($maxresults < 0){
			exit(0);
		}
	}
	else{
		$maxresults= -1;
	}

	$dictfile= $ARGV[1];



	$changeinsane= 0;

	@changeaddr= split /#/, $ARGV[2];

	if($#changeaddr < 1 || $#changeaddr > 2){
		print "binding address to be specified as IFACE#PREFIX#CHANGE\n";
	}
	elsif($#changeaddr == 2){
		if($changeaddr[2] =~ m/insane/){
			$changeinsane=1;
		}
	}

	($chaddr, $chpref) = split /\//, $changeaddr[1];

	if(!$chpref){
		$chpref=64;
	}

	$iface=$changeaddr[0];

	@revname=`uname`;
	$chos=$revname[0];

	if($chos =~ m/linux/i){
		$chos="linux";
	}
	elsif($chos =~ m/bsd/i){
		$chos="bsd";
	}
	else{
		print "Your OS is not supported for this function\n";
	}


	if($ARGV[0] eq "get-bing4-dict-batch"){
		print "Cannot specify source address for IPv4\n";
		exit;
	}
	elsif($ARGV[0] eq "get-bing6-dict-batch"){
		$ipprotocol=6;
	}
	else{
		# XXX: FOrce IPv6
		$ipprotocol=6;
	}
	ParseConfigFile();

	$SIG{INT} = \&SigIntHandler;

	if(!$changeinsane){
		$gotaddress=AddAddress($changeaddr[0], $chaddr, $chpref, $chos);
	}

	while(<STDIN>){


		undef @domain;

		if($_ =~ m/^\s*\.*([-a-zA-Z0-9.]+)/){
			$czone= $1;
		}
		else{
			next;
		}

		$czone= join ".", split /\./, $czone;

		print "######## DOMAIN: $czone ########\n";


		open my $dictfp, '<', $dictfile or die $!;

		if($changeinsane){
			$gotaddress=AddAddress($changeaddr[0], $chaddr, $chpref, $chos);
		}

		GetDomainsFromBingDict(\@domain, $czone, 0, $maxresults, $dictfp, $ipprotocol, $gotaddress);	

		if($changeinsane){
			DelAddress($changeaddr[0], $gotaddress, $chpref, $chos);
		}

		close $dictfp;

		#print "#### IMPORTANT: WILL NOW SEARCH FOR DOMAINS WITHIN SPECIFIC SUBDOMAINS\n";
		undef @remsites;
		undef @levsites;
		$continue=1;

		$domlevel=1;
		GenDomListLevel(\@domain, \@levsites, $czone, $domlevel, -1);


		while($#levsites >= 0 && $continue == 1){
			#print "#### LEVEL: $domlevel, TOTAL: $#levsites\n\n#LIST:\n";

			for($jj= 0; $jj <= $#levsites && $continue == 1; $jj++){
				$newczone= $levsites[$jj];

				#	open FILE, "$dictfile" or die $!;
				open my $dictfp, '<', $dictfile or die $!;

				if($changeinsane){
					$gotaddress=AddAddress($changeaddr[0], $chaddr, $chpref, $chos);
				}

				GetDomainsFromBingDict(\@domain, $newczone, $domlevel, $maxresults, $dictfp, $ipprotocol, $gotaddress);

				if($changeinsane){
					DelAddress($changeaddr[0], $gotaddress, $chpref, $chos);
				}

				if($maxresults >= 0 && $#domain>=$maxresults){
					$continue=0;
				}
				else{
					$continue=1;
				}

				close $dictfp;
			}

			$domlevel++;
			undef @levsites;
			GenDomListLevel(\@domain, \@levsites, $czone, $domlevel, -1);
		}  # While lev > 0;
	};  # while(<STDIN>)

	if(!$changeinsane){
		DelAddress($changeaddr[0], $gotaddress, $chpref, $chos);
	}


	exit;
}
elsif($ARGV[0] eq "get-bing-addr-dict-quick-batch" || $ARGV[0] eq "get-bing6-addr-dict-quick-batch"){
	if($#ARGV < 1 ){
		print "usage: script6 $ARGV[0] DICTIONARY ADDRESS [MAXRESULTS]\n";
		exit(1);
	}

	if($#ARGV > 2){
		$maxresults= $ARGV[3];
		$maxresults--;

		if($maxresults < 0){
			exit(0);
		}
	}
	else{
		$maxresults= -1;
	}

	$dictfile= $ARGV[1];



	$changeinsane= 0;

	@changeaddr= split /#/, $ARGV[2];

	if($#changeaddr < 1 || $#changeaddr > 2){
		print "binding address to be specified as IFACE#PREFIX#CHANGE\n";
	}
	elsif($#changeaddr == 2){
		if($changeaddr[2] =~ m/insane/){
			$changeinsane=1;
		}
	}

	($chaddr, $chpref) = split /\//, $changeaddr[1];

	if(!$chpref){
		$chpref=64;
	}

	$iface=$changeaddr[0];

	@revname=`uname`;
	$chos=$revname[0];

	if($chos =~ m/linux/i){
		$chos="linux";
	}
	elsif($chos =~ m/bsd/i){
		$chos="bsd";
	}
	else{
		print "Your OS is not supported for this function\n";
	}


	if($ARGV[0] eq "get-bing4-dict-quick-batch"){
		print "Cannot specify source address for IPv4\n";
		exit;
	}
	elsif($ARGV[0] eq "get-bing6-dict-quick-batch"){
		$ipprotocol=6;
	}
	else{
		# XXX: FOrce IPv6
		$ipprotocol=6;
	}

	ParseConfigFile();

	$SIG{INT} = \&SigIntHandler;

	if(!$changeinsane){
		$gotaddress=AddAddress($changeaddr[0], $chaddr, $chpref, $chos);
	}

	while(<STDIN>){


		undef @domain;

		if($_ =~ m/^\s*\.*([-a-zA-Z0-9.]+)/){
			$czone= $1;
		}
		else{
			next;
		}

		$czone= join ".", split /\./, $czone;

		print "######## DOMAIN: $czone ########\n";


		open my $dictfp, '<', $dictfile or die $!;

		if($changeinsane){
			$gotaddress=AddAddress($changeaddr[0], $chaddr, $chpref, $chos);
		}

		GetDomainsFromBingDict(\@domain, $czone, 0, $maxresults, $dictfp, $ipprotocol, $gotaddress);	

		if($changeinsane){
			DelAddress($changeaddr[0], $gotaddress, $chpref, $chos);
		}

		close $dictfp;
	};  # while(<STDIN>)

	if(!$changeinsane){
		DelAddress($changeaddr[0], $gotaddress, $chpref, $chos);
	}

	exit;
}

######################################
elsif($ARGV[0] eq "get-teoma-batch"){
	if($#ARGV >=1 ){
		$maxresults= $ARGV[1];
		$maxresults--;

		if($maxresults < 0){
			exit 0;
		}
	}
	else{
		$maxresults= -1;
	}

	ParseConfigFile();

	while(<STDIN>){


		undef @domain;

		if($_ =~ m/^\s*\.*([-a-zA-Z0-9.]+)/){
			$czone= $1;
		}
		else{
			next;
		}

		print "######## DOMAIN: $czone ########\n";


		GetDomainsFromTeoma(\@domain, $czone, 0, $maxresults);	

		#print "#### IMPORTANT: WILL NOW SEARCH FOR DOMAINS WITHIN SPECIFIC SUBDOMAINS\n";
		undef @remsites;
		undef @levsites;
		$continue=1;

		$domlevel=1;
		GenDomListLevel(\@domain, \@levsites, $czone, $domlevel, -1);


		while($#levsites >= 0 && $continue == 1){
			#print "#### LEVEL: $domlevel, TOTAL: $#levsites\n\n#LIST:\n";

			for($jj= 0; $jj <= $#levsites && $continue == 1; $jj++){
				$newczone= $levsites[$jj];

				GetDomainsFromTeoma(\@domain, $newczone, $domlevel, $maxresults);
				if($maxresults >= 0 && $#domain>=$maxresults){
					$continue=0;
				}
				else{
					$continue=1;
				}
			}

			$domlevel++;
			undef @levsites;
			GenDomListLevel(\@domain, \@levsites, $czone, $domlevel, -1);
		}  # While lev > 0;
	};  # while(<STDIN>)

	exit;
}
elsif($ARGV[0] eq "get-teoma"){
	if($#ARGV < 1 || $#ARGV > 2){
		print "Syntax: script6 get-teoma DOMAIN [MAXRESULTS]\n";
		exit 1;
	}

	if($#ARGV == 2){
		$maxresults= $ARGV[2];

		if($maxresults == 0){
			exit 0;
		}

		$maxresults--;
	}
	else{
		$maxresults= -1;
	}

	ParseConfigFile();

		undef @domain;

		$_= $ARGV[1];

		if($_ =~ m/^\s*\.*([-a-zA-Z0-9.]+)/){
			$czone= $1;
		}
		else{
			next;
		}

		print "######## DOMAIN: $czone ########\n";


		GetDomainsFromTeoma(\@domain, $czone, 0, $maxresults);	

		#print "#### IMPORTANT: WILL NOW SEARCH FOR DOMAINS WITHIN SPECIFIC SUBDOMAINS\n";
		undef @remsites;
		undef @levsites;
		$continue=1;

		$domlevel=1;
		GenDomListLevel(\@domain, \@levsites, $czone, $domlevel, -1);


		while($#levsites >= 0 && $continue == 1){
			#print "#### LEVEL: $domlevel, TOTAL: $#levsites\n\n#LIST:\n";

			for($jj= 0; $jj <= $#levsites && $continue == 1; $jj++){
				$newczone= $levsites[$jj];

				GetDomainsFromTeoma(\@domain, $newczone, $domlevel, $maxresults);
				if($maxresults >= 0 && $#domain>=$maxresults){
					$continue=0;
				}
				else{
					$continue=1;
				}
			}

			$domlevel++;
			undef @levsites;
			GenDomListLevel(\@domain, \@levsites, $czone, $domlevel, -1);
		}  # While lev > 0;

	exit;
}
elsif($ARGV[0] eq "get-duckduckgo"){
	if($#ARGV < 1 || $#ARGV > 2){
		print "Syntax: script6 get-duckduckgo DOMAIN [MAXRESULTS]\n";
		exit 1;
	}

	if($#ARGV == 2){
		$maxresults= $ARGV[2];

		if($maxresults == 0){
			exit 0;
		}

		$maxresults--;
	}
	else{
		$maxresults= -1;
	}

	ParseConfigFile();

		undef @domain;

		$_= $ARGV[1];

		if($_ =~ m/^\s*\.*([-a-zA-Z0-9.]+)/){
			$czone= $1;
		}
		else{
			next;
		}

		print "######## DOMAIN: $czone ########\n";


		GetDomainsFromDuckduckgo(\@domain, $czone, 0, $maxresults);	

		#print "#### IMPORTANT: WILL NOW SEARCH FOR DOMAINS WITHIN SPECIFIC SUBDOMAINS\n";
		undef @remsites;
		undef @levsites;
		$continue=1;

		$domlevel=1;
		GenDomListLevel(\@domain, \@levsites, $czone, $domlevel, -1);


		while($#levsites >= 0 && $continue == 1){
			#print "#### LEVEL: $domlevel, TOTAL: $#levsites\n\n#LIST:\n";

			for($jj= 0; $jj <= $#levsites && $continue == 1; $jj++){
				$newczone= $levsites[$jj];

				GetDomainsFromDuckduckgo(\@domain, $newczone, $domlevel, $maxresults);
				if($maxresults >= 0 && $#domain>=$maxresults){
					$continue=0;
				}
				else{
					$continue=1;
				}
			}

			$domlevel++;
			undef @levsites;
			GenDomListLevel(\@domain, \@levsites, $czone, $domlevel, -1);
		}  # While lev > 0;

	exit;
}
elsif($ARGV[0] eq "get-webcrawler-batch"){
	if($#ARGV >=1 ){
		$maxresults= $ARGV[1];
		$maxresults--;

		if($maxresults < 0){
			exit 0;
		}
	}
	else{
		$maxresults= -1;
	}

	ParseConfigFile();

	while(<STDIN>){


		undef @domain;

		if($_ =~ m/^\s*\.*([-a-zA-Z0-9.]+)/){
			$czone= $1;
		}
		else{
			next;
		}

		print "######## DOMAIN: $czone ########\n";


		GetDomainsFromWebcrawler(\@domain, $czone, 0, $maxresults);	

		#print "#### IMPORTANT: WILL NOW SEARCH FOR DOMAINS WITHIN SPECIFIC SUBDOMAINS\n";
		undef @remsites;
		undef @levsites;
		$continue=1;

		$domlevel=1;
		GenDomListLevel(\@domain, \@levsites, $czone, $domlevel, -1);


		while($#levsites >= 0 && $continue == 1){
			#print "#### LEVEL: $domlevel, TOTAL: $#levsites\n\n#LIST:\n";

			for($jj= 0; $jj <= $#levsites && $continue == 1; $jj++){
				$newczone= $levsites[$jj];

				GetDomainsFromWebcrawler(\@domain, $newczone, $domlevel, $maxresults);
				if($maxresults >= 0 && $#domain>=$maxresults){
					$continue=0;
				}
				else{
					$continue=1;
				}
			}

			$domlevel++;
			undef @levsites;
			GenDomListLevel(\@domain, \@levsites, $czone, $domlevel, -1);
		}  # While lev > 0;
	};  # while(<STDIN>)

	exit;
}
elsif($ARGV[0] eq "get-webcrawler"){
	if($#ARGV < 1 || $#ARGV > 2){
		print "Syntax: script6 get-webcrawler DOMAIN [MAXRESULTS]\n";
		exit 1;
	}

	if($#ARGV == 2){
		$maxresults= $ARGV[2];

		if($maxresults == 0){
			exit 0;
		}

		$maxresults--;
	}
	else{
		$maxresults= -1;
	}

	ParseConfigFile();

		undef @domain;

		$_= $ARGV[1];

		if($_ =~ m/^\s*\.*([-a-zA-Z0-9.]+)/){
			$czone= $1;
		}
		else{
			next;
		}

		print "######## DOMAIN: $czone ########\n";


		GetDomainsFromWebcrawler(\@domain, $czone, 0, $maxresults);	

		#print "#### IMPORTANT: WILL NOW SEARCH FOR DOMAINS WITHIN SPECIFIC SUBDOMAINS\n";
		undef @remsites;
		undef @levsites;
		$continue=1;

		$domlevel=1;
		GenDomListLevel(\@domain, \@levsites, $czone, $domlevel, -1);


		while($#levsites >= 0 && $continue == 1){
			#print "#### LEVEL: $domlevel, TOTAL: $#levsites\n\n#LIST:\n";

			for($jj= 0; $jj <= $#levsites && $continue == 1; $jj++){
				$newczone= $levsites[$jj];

				GetDomainsFromWebcrawler(\@domain, $newczone, $domlevel, $maxresults);
				if($maxresults >= 0 && $#domain>=$maxresults){
					$continue=0;
				}
				else{
					$continue=1;
				}
			}

			$domlevel++;
			undef @levsites;
			GenDomListLevel(\@domain, \@levsites, $czone, $domlevel, -1);
		}  # While lev > 0;

	exit;
}

# XXX
elsif($ARGV[0] eq "get-dns-zones"){
	@revname=`whoami`;
	$user= $revname[0];
	chomp($user);
	@revname=`hostname`;
	$hostname= $revname[0];
	chomp($hostname);

	
	while($line=<STDIN>){
		if($line=~ m/^\s*$/){
			next;
		}
		if($line=~ m/^\s*#/){
			print "$line";
		}
		elsif($line=~ m/^\s*([^#\s]+)/){
			$zone=$1;
			@revname=`date --iso-8601=seconds -u`;
			$datetime=$revname[0];
			$datetime=~ s/[ :+]//g;
			chomp($datetime);
			$filename="$zone-$datetime-$hostname-$user.txt";
			open FILE, ">$filename" or die $!;


			close FILE;
		}
	}
}
elsif($ARGV[0] eq "get-dns-root-zone" || $ARGV[0] eq "get-root-zone"){
	ParseConfigFile();

	do{
		# --user-agent "Mozilla/4.0"  works fine most of the time. curl || wget original headers cause trouble.
		$useragent=GetRandomUserAgent();

		@revname=`$wwwclient $wwwclient_sopt $wwwclient_agnt"$useragent" "https://www.iana.org/domains/root/db"`;

		if($? == 0){
			foreach $line (@revname){
				while($line =~ s/\<span class="domain tld"><a href="\/domains\/root\/db\/([^.>"]+).html">//){
					print "$1\n";
				}
			}
		}
	}while($? != 0);

	exit(0);
}
elsif($ARGV[0] eq "get-dns-root-zone-raw" || $ARGV[0] eq "get-root-zone-raw"){
	ParseConfigFile();

	do{
		# --user-agent "Mozilla/4.0"  works fine most of the time. curl || wget original headers cause trouble.
		$useragent=GetRandomUserAgent();

		@revname=`$wwwclient $wwwclient_sopt $wwwclient_agnt"$useragent" "https://data.iana.org/TLD/tlds-alpha-by-domain.txt"`;

		if($? == 0){
			foreach $line (@revname){
				while($line =~ s/^\s*(\S+)//){
					$domain=$1;
					print "$domain\n";
				}
			}
		}
	}while($? != 0);

	exit(0);
}
elsif($ARGV[0] eq "get-ukgov-zone"){
	ParseConfigFile();

	open FILE, "$dns_ukgov_database" or die $!;

	$line=<FILE>;

	if($line !~ m/﻿Domain Name,Owner,Representing/){
		print "Bad format in UK GOV zonefile $dns_ukgov_database\n";
		exit(1);
	}

	while($line=<FILE>){
		if($line =~ m/^\s*([-a-zA-Z0-9_.]+)/){
			print "$1\n";
		}
	}
}
elsif($ARGV[0] eq "get-asn" || $ARGV[0] eq "get-asn6"){
	@revname=`addr6 -a $ARGV[1] -r`;

	if($? != 0){
		print "Error in specified IPv6 address";
		exit(1);
	}

	chomp($revname[0]);
	$queryname= $revname[0] . ".origin6.asn.cymru.com.";
	@reverse=`host -t TXT $queryname`;

	if($? != 0){
		print "Error when trying to obtain ASN information\n";
		exit(1);
	}

	$reverse[0] =~ m/\"\s*((\d+)\s*)\s+|"/;

	if($1 eq ""){
		print "ASN unknown";
		exit(1);
	}

	print "$1\n";
	exit(0);
}
elsif($ARGV[0] eq "get-as" || $ARGV[0] eq "get-as6"){
	@revname=`addr6 -a $ARGV[1] -r`;

	if($? != 0){
		print "Error in specified IPv6 address\n";
		exit(1);
	}

	chomp($revname[0]);
	$queryname= $revname[0] . ".origin6.asn.cymru.com.";

	@reverse=`host -t TXT $queryname`;

	if($? != 0){
		print "Error when trying to obtain origin AS\n";
		exit(1);
	}

	$reverse[0] =~ m/\"(.*)\"/;

	if($1 eq ""){
		print "Origin ASN unknown\n";
	}
	else{
		print "$1\n";
	}

	$reverse[0] =~ m/"\s*(\d+).*"/;
	$asn=$1;
	$querystring="host -t TXT AS".$asn.".asn.cymru.com";
	@asinfo= `$querystring`;

	if($? != 0){
		print "Error when trying to obtain ASN information\n";
		exit(1);
	}

	$asinfo[0] =~ m/\"(.*)\"/;

	if($1 eq ""){
		print "Error when trying to obtain AS information";
	}
	else{
		print "$1\n";
	}

	exit(0);
}
elsif($ARGV[0] eq "ip6addr" || $ARGV[0] eq "ip6addrs" || $ARGV[0] eq "get-ip6addrs" || $ARGV[0] eq "get-ip6-addrs" || $ARGV[0] eq "get-ipv6-addrs" || $ARGV[0] eq "get-ipv6" || $ARGV[0] eq "get-ipv6-addr" || $ARGV[0] eq "get-ipv6-addresses"){
	while(<STDIN>){
		while($_ =~ s/((?:(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4})|(?:(?:(?:[0-9a-fA-F]{1,4}:){1,4}|:):(?:(?:(?:1[0-9]{2}|2(?:[0-4][0-9]|5[0-9])|[0-9]{1,2})\.){3}(?:(?:1[0-9]{2}|2(?:[0-4][0-9]|5[0-9])|[0-9]{1,2}))))|(?:(?:(?:(?:[0-9a-fA-F]{1,4}:)(?::[0-9a-fA-F]{1,4}){1,7})|(?:(?:[0-9a-fA-F]{1,4}:){2}(?::[0-9a-fA-F]{1,4}){1,6})|(?:(?:[0-9a-fA-F]{1,4}:){3}(?::[0-9a-fA-F]{1,4}){1,5})|(?:(?:[0-9a-fA-F]{1,4}:){4}(?::[0-9a-fA-F]{1,4}){1,4})|(?:(?:[0-9a-fA-F]{1,4}:){5}(?::[0-9a-fA-F]{1,4}){1,3})|(?:(?:[0-9a-fA-F]{1,4}:){6}(?::[0-9a-fA-F]{1,4}){1,2})|(?:[0-9a-fA-F]{1,4}:){1,7}:)|(?::(?:(?::[0-9a-fA-F]{1,4}){1,7}|:))))//){
			print "$1\n";
		}
	}
}
elsif($ARGV[0] eq "ipaddr" || $ARGV[0] eq "ipaddrs" || $ARGV[0] eq "get-ipaddrs" || $ARGV[0] eq "get-ip-addrs" || $ARGV[0] eq "ipv4addr" || $ARGV[0] eq "get-ipv4" || $ARGV[0] eq "ipv4addrs" || $ARGV[0] eq "get-ipv4addrs" || $ARGV[0] eq "get-ipv4-addrs" || $ARGV[0] eq "get-ipv4-addresses"){
	while(<STDIN>){
#print "lei: $_";
		while($_ =~ s/((?:(?:1[0-9]{2}|2(?:[0-4][0-9]|5[0-9])|[0-9]{1,2})\.){3}(?:(?:1[0-9]{2}|2(?:[0-4][0-9]|5[0-9])|[0-9]{1,2})))//){
#		while($_ =~ s/((((?:1[0-9]{2}|2(?:[0-4][0-9]|5[0-9])|[0-9]{1,2})\.){3}(?:(?:1[0-9]{2}|2(?:[0-4][0-9]|5[0-9])|[0-9]{1,2}))))//){
			print "$1\n";
		}
	}
}
elsif($ARGV[0] eq "ippref" || $ARGV[0] eq "ipprefs" || $ARGV[0] eq "get-ipprefs" || $ARGV[0] eq "get-ip-prefs" || $ARGV[0] eq "ipv4pref" || $ARGV[0] eq "ipv4prefs" || $ARGV[0] eq "get-ipv4prefs" || $ARGV[0] eq "get-ipv4-prefs" || $ARGV[0] eq "get-ipv4-prefixes"){
	while(<STDIN>){
		while($_ =~ s/((?:(?:(?:1[0-9]{2}|2(?:[0-4][0-9]|5[0-9])|[0-9]{1,2})\.){3}(?:(?:1[0-9]{2}|2(?:[0-4][0-9]|5[0-9])|[0-9]{1,2})))\/(3[0-2]|[1-2]?[0-9]))//){
			print "$1\n";
		}
	}
}
elsif($ARGV[0] eq "ipv6pref" || $ARGV[0] eq "ipv6prefs" || $ARGV[0] eq "get-ipv6prefs" || $ARGV[0] eq "get-ip6-prefs" || $ARGV[0] eq "get-ipv6-prefs" || $ARGV[0] eq "ipv6prefix" || $ARGV[0] eq "ipv6prefixes" || $ARGV[0] eq "get-ipv6prefixes" || $ARGV[0] eq "get-ip6-prefixes" || $ARGV[0] eq "get-ipv6-prefixes"){
	while(<STDIN>){
		while($_ =~ s/((?:(?:(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4})|(?:(?:(?:[0-9a-fA-F]{1,4}:){1,4}|:):(?:(?:(?:1[0-9]{2}|2(?:[0-4][0-9]|5[0-9])|[0-9]{1,2})\.){3}(?:(?:1[0-9]{2}|2(?:[0-4][0-9]|5[0-9])|[0-9]{1,2}))))|(?:(?:(?:(?:[0-9a-fA-F]{1,4}:)(?::[0-9a-fA-F]{1,4}){1,7})|(?:(?:[0-9a-fA-F]{1,4}:){2}(?::[0-9a-fA-F]{1,4}){1,6})|(?:(?:[0-9a-fA-F]{1,4}:){3}(?::[0-9a-fA-F]{1,4}){1,5})|(?:(?:[0-9a-fA-F]{1,4}:){4}(?::[0-9a-fA-F]{1,4}){1,4})|(?:(?:[0-9a-fA-F]{1,4}:){5}(?::[0-9a-fA-F]{1,4}){1,3})|(?:(?:[0-9a-fA-F]{1,4}:){6}(?::[0-9a-fA-F]{1,4}){1,2})|(?:[0-9a-fA-F]{1,4}:){1,7}:)|(?::(?:(?::[0-9a-fA-F]{1,4}){1,7}|:))))\/(1([0-1][0-9]|2[0-8])|[1-9]?[0-9]))//){
			print "$1\n";
		}
	}
}
elsif($ARGV[0] eq "get-trace-stats" || $ARGV[0] eq "get-trace6-stats" || $ARGV[0] eq "get-trace-results"){
	$valid=0;
	$dropped=0;
	$pass=0;
	$total=0;
	$sameas=0;
	$difas=0;
	$sameas2=0;
	$difas2=0;
	$unknownas=0;
	$unknownipdrop=0;
	$unknownipfree=0;
	$unknowniptrouble=0;
	$ipfreedrop=0;
	$hoperror=0;
	$hopzero=0;
	$unknownas=0;
	$unknowniporigas=0;
	$unknownipdropas=0;

	print "$SI6_TOOLKIT\n";
	print "$SCRIPT6\n";
	print "[Computing stats....]\n\n";

	while($fline=<STDIN>){
		chomp($fline);

		if($fline =~ m/([^\x0d]+)/){
			$fline= $1;
		}

		# Check whether this line is just a comment -- if so, skip it
		if($fline =~ m/^((\s)*#)/){
			next;
		}

		# Skip blanck lines 
		if($fline =~ m/^(\s)*$/){
			next;
		}

		# Grab elements 
		if($fline =~ m/(\S*)#(\S*)#(\d*)#(\S*)#(\d*)#(\S*)#(\S*)/){
			$total++;

			$iporig= $1;
			$ipfree= $2;
			$hopsfree= $3;
			$iptrouble= $4;
			$hopstrouble= $5;
			$ipdrop=$6;
			$ipdrop2=$7;

			# Discard this line if we did not learn any host address in the non-EH path6
			if($ipfree eq ""){
				$unknownipfree++;
				next;
			}

			# Discard this line if we did not learn any address in the path6 with EHs
			if($iptrouble eq ""){
				$unknowniptrouble++;
				next;
			}

			# If the final node in the normal traceroute was not the final destination, discard
			# this line -- it is impossible to learn where the packets are being dropped
			if($ipfree ne $iporig){
				$ipfreedrop++;
				next;
			}

			
			# The number of hops from the first-responding node cannot possibly be 0.
			if($hopsfree == 0){
				$hopzero++;
				next;
			}

			# The number of hops till the last responding node cannot be larger for the
			# EH-case than for the non-EH case.
			if($hopstrouble > $hopsfree){
				$hoperror++;
				next;
			}

			# If packets are being dropped but the address of the node dropping the packets is
			# unavailable, discard this line
			if($iporig ne $iptrouble && $ipdrop eq ""){
				$unknownipdrop++;
				next;
			}

			# If the line has passed all the previous checks, we consider it to be valid
			$valid= $valid+1;
			$delta= $hopsfree - $hopstrouble;

			# If the last responding node of the EH case is the destination address, then there
			# are no packet drops
			if($iporig eq $iptrouble){
				$pass=$pass+1;
			}
			else{
				# Otherwise, there is a packet drop

				$dropped= $dropped +1;
				$hops[$delta]++;

				$asiporig= GetASN($iporig);

				if($asiporig eq ""){
					$unknownas2++;
					$unknownas++;
					$unknowniporigas++;
					next;
				}

				$asipdrop= GetASN($ipdrop);

				if($asipdrop eq ""){
					$unknownas++;
					$unknownipdropas++;
				}
				else{
					# Count whether the dropping node is in the same or different ASN as the destination node (worst-case)
					if($asiporig == $asipdrop){
						$sameas++;

						if($destasworst[$asipdrop] == 0){
							$numdestasworst++;
							$destasorgworst[$asipdrop]= GetORG($asipdrop);

							if($destasorgworst[$asipdrop] eq ""){
								$destasorgworst[$asipdrop]="Unknown";
							}
						}

						$ndestasworst++;
						$destasworst[$asipdrop]++;
					}
					else{
						$difas++;

						if($nondestasworst[$asipdrop] == 0){
							$numnondestasworst++;
							$nondestasorgworst[$asipdrop]= GetORG($asipdrop);

							if($nondestasorgworst[$asipdrop] eq ""){
								$nondestasorgworst[$asipdrop]="Unknown";
							}
						}

						$nnondestasworst++;
						$nondestasworst[$asipdrop]++;
					}
				}

				if($ipdrop2 ne ""){
					$asipdrop2= GetASN($ipdrop2);

					if($asipdrop2 eq ""){
						$unknownas2++;
						$unknownipdropas2++;
						next;
					}
				}
				else{
					# If M+2 is "null", we use M+1
					if($asipdrop ne ""){
						$asipdrop2= $asipdrop;
					}
					else{
						$unknownas2++;
						$unknownipdropas2++;
						next;
					}
				}

				if($asiporig == $asipdrop2){
					$sameas2++;

					if($destasbest[$asipdrop2] == 0){
						$numdestasbest++;

						$destasorgbest[$asipdrop2]= GetORG($asipdrop2);

						if($destasorgbest[$asipdrop2] eq ""){
							$destasorgbest[$asipdrop2]="Unknown";
						}
					}

					$ndestasbest++;
					$destasbest[$asipdrop2]++;
				}
				else{
					$difas2++;

					if($nondestasbest[$asipdrop2] == 0){
						$numnondestasbest++;
						$nondestasorgbest[$asipdrop2]= GetORG($asipdrop2);

						if($nondestasorgbest[$asipdrop2] eq ""){
							$nondestasorgbest[$asipdrop2]="Unknown";
						}
					}

					$nnondestasbest++;
					$nondestasbest[$asipdrop2]++;
				}
			}
		}
	}

#	print "Total entries: $total\t Valid entries: $valid\n";

	if($valid >= 1){
		$passp= ($pass * 100)/$valid;
		$droppedp= ($dropped * 100)/$valid;

		printf "Valid entries: %d\tPassed: %d (%.2f%%)\t Dropped: %d (%.2f%%)\n", $valid, $pass, $passp, $dropped, $droppedp;
	}
	else{
		print "No valid entries were found\n";
	}

	if($dropped >= 1){
		print "******************************************************************************\n";
		$sameasp= ($sameas * 100)/$dropped;
		$difasp= ($difas * 100)/$dropped;
		$unknownasp= ($unknownas * 100)/$dropped;
		$sameasp2= ($sameas2 * 100)/$dropped;
		$difasp2= ($difas2 * 100)/$dropped;
		$unknownasp2= ($unknownas2 * 100)/$dropped;

		print "PACKET DROPS\n\n";

		if($sameas == $sameas2 && $difas == $difas2){
			printf "Same ASN: %d (%.2f%%)\t Different ASN: %d (%.2f%%)\t Unknown ASN:%d (%.2f%%)\n", $sameas, $sameasp, $difas, $difasp, $unknownas, $unknownasp;
		}
		else{
			printf "Best case:\nSame ASN: %d (%.2f%%)\t Different ASN: %d (%.2f%%)\t Unknown ASN:%d (%.2f%%)\n", $sameas2, $sameasp2, $difas2, $difasp2, $unknownas2, $unknownasp2;
			printf "Worst case:\nSame ASN: %d (%.2f%%)\t Different ASN: %d (%.2f%%)\t Unknown ASN:%d (%.2f%%)\n", $sameas, $sameasp, $difas, $difasp, $unknownas, $unknownasp;
		}

		print "******************************************************************************\n";
		print "DELTA-HOPS DISTRIBUTION\n\n";
		for($i=0; $i<= $#hops; $i++){
			if($hops[$i] != 0){
				$dhop= ($hops[$i] * 100)/$dropped;
				printf "Delta-Hop %2d: %6d (%.2f%%)\n", $i, $hops[$i], $dhop;
			}
		}

		print "******************************************************************************\n";
		print "AUTONOMOUS SYSTEMS DISTRIBUTION\n";
		printf "\nNon-Destination Dropping AS Distribution (best case): (%u ASes)\n\n", $numnondestasbest;

		if($numnondestasbest == 0){
			print "(no ASes)\n";
		}

		for($i=0; $i<= $#nondestasbest; $i++){
			if($nondestasbest[$i] != 0){
#				($nondestasbest[$i] * 100)/$nnondestasbest;
				printf "%6d (%.2f%%): AS %5d (%s)\n", $nondestasbest[$i], ($nondestasbest[$i] * 100)/$nnondestasbest, $i, $nondestasorgbest[$i];
			}
		}

		printf "\nDestination Dropping AS Distribution (best case): (%u ASes)\n\n", $numdestasbest;

		if($numdestasbest == 0){
			print "(no ASes)\n";
		}

		for($i=0; $i<= $#destasbest; $i++){
			if($destasbest[$i] != 0){
#				($destasbest[$i] * 100)/$ndestasbest;

				printf "%6d (%.2f%%): AS %5d (%s)\n", $destasbest[$i], ($destasbest[$i] * 100)/$ndestasbest, $i, $destasorgbest[$i];
			}
		}

		printf "\nNon-Destination Dropping AS Distribution (worst case): (%u ASes)\n\n", $numnondestasworst;

		if($numnondestasworst == 0){
			print "(no ASes)\n";
		}

		for($i=0; $i<= $#nondestasworst; $i++){
			if($nondestasworst[$i] != 0){
#				($nondestasworst[$i] * 100)/$nnondestasworst;
				printf "%6d (%.2f%%): AS %5d (%s)\n", $nondestasworst[$i], ($nondestasworst[$i] * 100)/$nnondestasworst, $i, $nondestasorgworst[$i];
			}
		}

		printf "\nDestination Dropping AS Distribution (worst case): (%u ASes)\n\n", $numdestasworst;

		if($numdestasworst == 0){
			print "(no ASes)\n";
		}

		for($i=0; $i<= $#destasworst; $i++){
			if($destasworst[$i] != 0){
#				($destasworst[$i] * 100)/$ndestasworst;
				printf "%6d (%.2f%%): AS %5d (%s)\n", $destasworst[$i], ($destasworst[$i] * 100)/$ndestasworst, $i, $destasorgworst[$i];
			}
		}
	}

	if( ($total - $valid) > 0){
		print "******************************************************************************\n";
		print "VALIDATION OF INPUT DATA\n\n";
		printf "Total entries: %d\tValid entries: %d\tInvalid: %d\n", $total, $valid, ($total-$valid);
		print "Unknown no-EH IPv6: $unknownipfree\tUnknown EH IPv6: $unknowniptrouble\tNo-EH IPv6 != Dst IPv6: $ipfreedrop\n";
		print "No-EH Hops==0: $hopzero\tNo-EH Hops < EH Hops: $hoperror\n";
	}
}
elsif($ARGV[0] eq "get-trace-stats-old" || $ARGV[0] eq "get-trace6-stats-old"){
	$valid=0;
	$dropped=0;
	$pass=0;
	$total=0;
	$sameas=0;
	$difass=0;
	$unknownas=0;
	$unknownipdrop=0;
	$unknownipfree=0;
	$unknowniptrouble=0;
	$ipfreedrop=0;
	$hoperror=0;
	$hopzero=0;
	$unknownas=0;
	$unknowniporigas=0;
	$unknownipdropas=0;

	while($fline=<STDIN>){
		chomp($fline);

		if($fline =~ m/([^\x0d]+)/){
			$fline= $1;
		}

		# Check whether this line is just a comment -- if so, skip it
		if($fline =~ m/^((\s)*#)/){
			next;
		}

		# Skip blanck lines 
		if($fline =~ m/^(\s)*$/){
			next;
		}

		# Grab elements 
		if($fline =~ m/(\S*)#(\S*)#(\d*)#(\S*)#(\d*)#(\S*)/){
			$total++;

			$iporig= $1;
			$ipfree= $2;
			$hopsfree= $3;
			$iptrouble= $4;
			$hopstrouble= $5;
			$ipdrop=$6;

			if($ipfree eq ""){
				$unknownipfree++;
				next;
			}

			if($iptrouble eq ""){
				$unknowniptrouble++;
				next;
			}

			if($ipfree ne $iporig){
				$ipfreedrop++;
				next;
			}

			if($hopsfree == 0){
				$hopzero++;
				next;
			}

			if($hopstrouble > $hopsfree){
				$hoperror++;
				next;
			}

			$delta= $hopsfree - $hopstrouble;


			$valid= $valid+1;

			if($iporig eq $iptrouble){
				#notdropped
				$pass=$pass+1;
			}
			else{
				$dropped= $dropped +1;
				$hops[$delta]++;

				if($ipdrop eq ""){
					$unknownipdrop++;
					$unknownas++;
					next;
				}
				else{
					undef @revname;
					undef @reverse;
					@revname=`addr6 -a $iporig -r`;
					chomp($revname[0]);
					$queryname= $revname[0] . ".origin6.asn.cymru.com.";

					@reverse=`host -t TXT $queryname`;
					$reverse[0] =~ m/\"\s*((\d+)\s*)\s+|"/;

					$asiporig=$2;

					if($asiporig==0){
						$unknownas++;
						$unknowniporigas++;
						next;
					}

					undef @revname;
					undef @reverse;
					@revname=`addr6 -a $ipdrop -r`;
					chomp($revname[0]);
					$queryname= $revname[0] . ".origin6.asn.cymru.com.";

					@reverse=`host -t TXT $queryname`;
					$reverse[0] =~ m/\"\s*((\d+)\s*)\s+|"/;
					$asipdrop=$1;

					if($asipdrop==0){
						$unknownas++;
						$unknownipdropas++;
						next;
					}

					if($asiporig eq "" || $asipdrop eq ""){
						$unknownas++;
					}

					if($asiporig == $asipdrop){
						$sameas++;
					}
					else{
						$difas++;
					}
				}
			}

	#		print "iporig: $1, ipfree: $2, hopsfree: $3, iptrouble: $4, hopstrouble: $5\n";
		}

	}

	print "$SI6_TOOLKIT\n";
	print "$SCRIPT6\n\n";

#	print "Total entries: $total\t Valid entries: $valid\n";

	if($valid >= 1){
		$passp= ($pass * 100)/$valid;
		$droppedp= ($dropped * 100)/$valid;

		printf "Valid entries: %d\tPass: %d (%.2f%%)\t Dropped: %d (%.2f%%)\n", $valid, $pass, $passp, $dropped, $droppedp;
		print "******************************************************************************\n";
	}
	else{
		print "No valid entries were found\n";
	}

	if($dropped >= 1){
		$sameasp= ($sameas * 100)/$dropped;
		$difasp= ($difas * 100)/$dropped;
		$unknownasp= ($unknownas * 100)/$dropped;

		print "Packet Drops\n\n";
		printf "Same ASN: %d (%.2f%%)\t Different ASN: %d (%.2f%%)\t Unknown ASN:%d (%.2f%%)\n\n", $sameas, $sameasp, $difas, $difasp, $unknownas, $unknownasp;

		print "Delta-Hops distribution\n\n";
		for($i=0; $i<= $#hops; $i++){
			if($hops[$i] != 0){
				$dhop= ($hops[$i] * 100)/$dropped;
				printf "DHop %d: %d (%.2f%%)\n", $i, $hops[$i], $dhop;
			}
		}
	}

	print "******************************************************************************\n";
	if( ($total - $valid) > 0){
		print "Validation of input data\n\n";
		printf "Total entries: %d\tValid entries: %d\tInvalid: %d\n", $total, $valid, ($total-$valid);
		print "Unknown no-EH IPv6: $unknownipfree\tUnknown EH IPv6: $unknowniptrouble\tNo-EH IPv6 != Dst IPv6: $ipfreedrop\n";
		print "No-EH Hops==0: $hopzero\tNo-EH Hops < EH Hops: $hoperror\n";
	}
}
elsif($ARGV[0] eq "get-trace-invalid" || $ARGV[0] eq "get-trace6-invalid"){
	$valid=0;
	$dropped=0;
	$pass=0;
	$total=0;
	$sameas=0;
	$difass=0;
	$unknownas=0;
	$unknownipdrop=0;
	$unknownipfree=0;
	$unknowniptrouble=0;
	$ipfreedrop=0;
	$hoperror=0;
	$hopzero=0;
	$unknownas=0;

	while($fline=<STDIN>){
		chomp($fline);

		if($fline =~ m/([^\x0d]+)/){
			$fline= $1;
		}

		# Check whether this line is just a comment -- if so, skip it
		if($fline =~ m/^((\s)*#)/){
			next;
		}

		# Skip blanck lines 
		if($fline =~ m/^(\s)*$/){
			next;
		}

		# Grab elements 
		if($fline =~ m/(\S*)#(\S*)#(\d*)#(\S*)#(\d*)#(\S*)/){
			$total++;

			$iporig= $1;
			$ipfree= $2;
			$hopsfree= $3;
			$iptrouble= $4;
			$hopstrouble= $5;
			$ipdrop=$6;

			if($ipfree eq ""){
				$unknownipfree++;
				print "$fline\n";
				next;
			}

			if($iptrouble eq ""){
				$unknowniptrouble++;
				print "$fline\n";
				next;
			}

			if($ipfree ne $iporig){
				$ipfreedrop++;
				print "$fline\n";
				next;
			}

			if($hopsfree == 0){
				$hopzero++;
				print "$fline\n";
				next;
			}

			if($hopstrouble > $hopsfree){
				$hoperror++;
				print "$fline\n";
				next;
			}
		}
	}
}
elsif($ARGV[0] eq "get-alexa-domains"){
	while($line=<STDIN>){
		while($line =~ m{\d+,([^/\s]*)}ig){
			print "$1\n";		
		}
	}
}
elsif($ARGV[0] eq "get-majestic-domains"){
	while($line=<STDIN>){
		while($line =~ m{\d+,\d+,([^/\s,]*),.*}ig){
			print "$1\n";		
		}
	}
}
elsif($ARGV[0] eq "trace" || $ARGV[0] eq "trace6" || $ARGV[0] eq "get-trace" || $ARGV[0] eq "get-trace6"){
	if($> != 0){
		print "Error: This script requires superuser privileges\n";
		exit 1;
	}

	select STDOUT; $| = 1;

	$total=0;
# XXX
#	$response=0;
#	$timeout=0;

	$ehtype= "do";
	$ehsize= "8";
	$prototype="icmp";

	if($#ARGV > 0){
		if($ARGV[1] =~ m/([a-zA-Z]+):?(\d+)/){
			$ehtype= lc($1);
			$ehsize= $2;
		}
		else{
			$ehtype= $ARGV[1];
		}

		if($#ARGV > 1){
			$prototype=	$ARGV[2];

			if($#ARGV > 2){
				$port=	$ARGV[3];
			}
			else{
				$port= "80";
			}
		}
	}

	if($prototype eq "tcp"){
		$protoopt= "--tcp-flags S -a $port";
	}
	else{
		$protoopt="";
	}

	if($ehtype eq "fh" || $ehtype eq "frag"){
		if($ehsize < 8){
			print "Error: Fragment size should be larger than or equal to 8 bytes";
			exit 1;
		}
		elsif($ehsize > 1280){
			print "Error: Fragment size should be smaller than or equal to 1280 bytes";
			exit 1;
		}

		$payload= ($ehsize * 2) - 20;
		$protoopt="-P $payload " . $protoopt;
	}

	if($ehtype eq "fh" || $ehtype eq "frag"){
		$eh= "-y $ehsize";
	}
	elsif($ehtype eq "hbh"){
		$eh= "-H $ehsize";
	}
	elsif($ehtype eq "do"){
		$eh= "-u $ehsize";
	}
	elsif($ehtype eq "ah"){
		$eh= "-p ah";
	}
	elsif($ehtype eq "esp"){
		$eh= "-p esp";
	}
	else{
		print "Error: Unknown EH type";
		exit 1;
	}

#	print "path6 -d NOSPEC -p $prototype $protoopt --rate-limit 20pps\n";
#	print "path6 -d NOSPEC -p $prototype $protoopt $eh --rate-limit 20pps\n";

	while($fline=<STDIN>){
		chomp($fline);

		if($fline =~ m/([^\x0d]+)/){
			$fline= $1;
		}

		# Check whether this line is just a comment -- if so, skip it
		if($fline =~ m/^((\s)*#)/){
			next;
		}

		# Skip blanck lines 
		if($fline =~ m/^(\s)*$/){
			next;
		}

		# Remove all items from the array of IP addresses
		undef @ipsfree;
		undef @tcp;

		$maxhopsfree=0;
		$maxhopsfreeip="";
		@tcp=`path6 -d $fline -p $prototype $protoopt`; # --rate-limit 25pps

		if($? != 0){
			next;
		}

		foreach $line (@tcp){
			# Discard lines that do not contain a "probe" line
			if($line =~ m/\s+(\d+)\s+\((\S*)\)/){
				if($1 > $maxhopsfree){
					if($2 ne ""){
						$maxhopsfree=$1;
						$maxhopsfreeip=$2;
					}

					# We store the IPv6 addresses of all hops
					push(@ipsfree, $2);
				}
			}
		}

		undef @tcp;
		$maxhopstrouble=0;
		$maxhopstroubleip="";

		# XXX: This is an ugly hack. should be removed. ESP and AH should possibly be handled as EHs, rather than probe types
		if($ehtype eq "esp" || $ehtype eq "ah"){
			@tcp=`path6 -d $fline $eh`; # --rate-limit 25pps
		}else{
			@tcp=`path6 -d $fline -p $prototype $protoopt $eh`; #  --rate-limit 25pps
		}

		if($? != 0){
			next;
		}

		foreach $line (@tcp){
			# Discard lines that do not contain a "probe" line
			if($line =~ m/\s+(\d+)\s+\((\S*)\)/){
				if($1 > $maxhopstrouble){
					if($2 ne ""){
						$maxhopstrouble=$1;
						$maxhopstroubleip= $2;
					}
				}
			}
		}

		$dropip="";
		$dropip2="";

		for($i=0; $i<= $#ipsfree; $i++){
			if($ipsfree[$i] eq $maxhopstroubleip){
				if($i == $#ipsfree){
					$dropip= $ipsfree[$i];
				}
				else{
					$dropip= $ipsfree[$i+1];
				}

				if($dropip eq $fline){
					$dropip2= $dropip;
				}
				else{
					$dropip2= $ipsfree[$i+2];
					if($dropip2 eq ""){
						$dropip2= $dropip;
					}
				}

				last;
			}
		}

		print "$fline#$maxhopsfreeip#$maxhopsfree#$maxhopstroubleip#$maxhopstrouble#$dropip#$dropip2\n";

	}
}
elsif($ARGV[0] eq "get-routers-traceroute6" || $ARGV[0] eq "get-routers" || $ARGV[0] eq "get-ipv6-routers"){
	$| = 1;
	while(<STDIN>){
		if($_ =~ m/^\s*#/){
				print "$_";
			next;
		}

		# Get rid of spaces, and also leave out in-line comments
		if($_ =~ m/\s*([^#\s]+)/){
			$ipname=$1;


#			if( $ip !~ m/(?:(?:(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4})|(?:(?:(?:[0-9a-fA-F]{1,4}:){1,4}|:):(?:(?:(?:1[0-9]{2}|2(?:[0-4][0-9]|5[0-9])|[0-9]{1,2})\.){3}(?:(?:1[0-9]{2}|2(?:[0-4][0-9]|5[0-9])|[0-9]{1,2}))))|(?:(?:(?:(?:[0-9a-fA-F]{1,4}:)(?::[0-9a-fA-F]{1,4}){1,7})|(?:(?:[0-9a-fA-F]{1,4}:){2}(?::[0-9a-fA-F]{1,4}){1,6})|(?:(?:[0-9a-fA-F]{1,4}:){3}(?::[0-9a-fA-F]{1,4}){1,5})|(?:(?:[0-9a-fA-F]{1,4}:){4}(?::[0-9a-fA-F]{1,4}){1,4})|(?:(?:[0-9a-fA-F]{1,4}:){5}(?::[0-9a-fA-F]{1,4}){1,3})|(?:(?:[0-9a-fA-F]{1,4}:){6}(?::[0-9a-fA-F]{1,4}){1,2})|(?:[0-9a-fA-F]{1,4}:){1,7}:)|(?::(?:(?::[0-9a-fA-F]{1,4}){1,7}|:))))/){
#				next;
#			}

			# Obtain the IPv6 addres corresponding to the specified domain name
			# (This implicitly obtains the canonic address if an IPv6 address was specified)
			my ( $erraddr, @addrs ) = Socket::getaddrinfo( $ipname, 0, { 'protocol' => Socket::IPPROTO_TCP, 'family' => Socket::AF_INET6 } );

			if($erraddr){
				die "getadrinfo(): $erraddr";
			}

			my ( $errname, $ip ) = Socket::getnameinfo( $addrs[0]->{addr}, Socket::NI_NUMERICHOST );

			if ($errname){
				die "getnameinfo() $errname";
			}

			# At this point we have one IPv6 address to traceroute to (in $IP)

			# Open pipe to addr6, to filter out duplicate addresses.
			open FILTER, "| addr6 -i -q" or die "can't use addr6 for filtering addresses: $!";

			@revname=`traceroute6 -n $ip 2>&1`;

			if($? == 0){
				print "# $ip\n";

				foreach $line (@revname){
					while($line =~ s/((?:(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4})|(?:(?:(?:[0-9a-fA-F]{1,4}:){1,4}|:):(?:(?:(?:1[0-9]{2}|2(?:[0-4][0-9]|5[0-9])|[0-9]{1,2})\.){3}(?:(?:1[0-9]{2}|2(?:[0-4][0-9]|5[0-9])|[0-9]{1,2}))))|(?:(?:(?:(?:[0-9a-fA-F]{1,4}:)(?::[0-9a-fA-F]{1,4}){1,7})|(?:(?:[0-9a-fA-F]{1,4}:){2}(?::[0-9a-fA-F]{1,4}){1,6})|(?:(?:[0-9a-fA-F]{1,4}:){3}(?::[0-9a-fA-F]{1,4}){1,5})|(?:(?:[0-9a-fA-F]{1,4}:){4}(?::[0-9a-fA-F]{1,4}){1,4})|(?:(?:[0-9a-fA-F]{1,4}:){5}(?::[0-9a-fA-F]{1,4}){1,3})|(?:(?:[0-9a-fA-F]{1,4}:){6}(?::[0-9a-fA-F]{1,4}){1,2})|(?:[0-9a-fA-F]{1,4}:){1,7}:)|(?::(?:(?::[0-9a-fA-F]{1,4}){1,7}|:))))//){
						if($1 ne $ip){
							print FILTER "$1\n";
						}
					}
				}
			}

			close FILTER;
		}

	}
}
elsif($ARGV[0] eq "get-domain-level"){

	if($#ARGV != 1){
		print "Usage: script6 get-domain-level LEVEL\n";
		exit(1);
	}

	$level= $ARGV[1];

	if($level < 1){
		print "Error: Invalid domain level\n";
		exit(1);
	}

	$level=$level-1;

	while($domain=<STDIN>){
		chomp($domain);

		if( $domain =~ m/^\s*([-a-zA-Z0-9_.]+)/){
			$domain= $1;

			@revarray=();
	#		$revdomain= join(".", reverse split /\./, $domain);
			@revarray= reverse split /\./, $domain;

			if($#revarray >= $level){
				$revdomain= join(".", @revarray[0..$level]);
				$grabbeddomain= join(".", reverse split /\./, $revdomain);

				for($i=0; $i<= $#domains; $i++){
					if($domains[$i] eq $grabbeddomain){
						last;
					}
				}

				if($i > $#domains){
					$domains[$#domains + 1]=$grabbeddomain;
					print "$grabbeddomain\n";
				}
			}
		}
	}
}
elsif($ARGV[0] eq "create-dict" || $ARGV[0] eq "create-dictionary" || $ARGV[0] eq "create-dic"){
	@keywords=(); 

	while($line=<STDIN>){
		chomp($line);

		if( $line =~ m/^\s*([^\/#\r\n]+)/){
			$dictword=$1;
			$dictword=~ s/\r//g;
	#		$dictword=~ s/ /%20/g;

			for($i=0; $i<= $#keywords; $i++){
				if($keywords[$i] eq $dictword){
					last;
				}
			}

			if($i > $#keywords){
				$keywords[$#keywords + 1]=$dictword;
				print "$dictword\n";
			}
		}
	}
}
elsif($ARGV[0] eq "create-dict-no-spaces" || $ARGV[0] eq "create-dictionary-no-spaces" || $ARGV[0] eq "create-dic-no-spaces"){
	@keywords=(); 

	while($line=<STDIN>){
		chomp($line);

		if( $line =~ m/^\s*([^\/#\r\n\s]+)/){
			$dictword=$1;
			$dictword=~ s/\r//g;
	#		$dictword=~ s/ /%20/g;

			for($i=0; $i<= $#keywords; $i++){
				if($keywords[$i] eq $dictword){
					last;
				}
			}

			if($i > $#keywords){
				$keywords[$#keywords + 1]=$dictword;
				print "$dictword\n";
			}
		}
	}
}
elsif($ARGV[0] eq "create-dict-discarded" || $ARGV[0] eq "create-dictionary-discarded" || $ARGV[0] eq "create-dic-discarded"){
	@keywords=(); 

	while($line=<STDIN>){
		chomp($line);

		if( $line =~ m/^\s*([^\/0-9#\n]+)/){
			$dictword=$1;

			for($i=0; $i<= $#keywords; $i++){
				if($keywords[$i] eq $dictword){
					last;
				}
			}

			if($i > $#keywords){
				$keywords[$#keywords + 1]=$dictword;
			}
			else{
				print "$dictword\n";
			}
		}
		else{
			print "#$dictword (didnt match regex)\n";
		}
	}
}
elsif($ARGV[0] eq "get-unique-domains"){
	@keywords=(); 

	while($line=<STDIN>){
		chomp($line);

		if( $line =~ m/^\s*([-a-zA-Z0-9_.]+)/){
			$dictword=$1;
			$dictword=~ s/\r//g;
			$dictword=lc($dictword);
			$dictword=~ s/\.$//;

	#		$dictword=~ s/ /%20/g;

			for($i=0; $i<= $#keywords; $i++){
				if($keywords[$i] eq $dictword){
					last;
				}
			}

			if($i > $#keywords){
				$keywords[$#keywords + 1]=$dictword;
				print "$dictword\n";
			}
		}
	}
}
elsif($ARGV[0] eq "get-unique-ipv4-addresses" || $ARGV[0] eq "get-unique-ipv4-addrs"){
	@keywords=(); 

	while($line=<STDIN>){
		chomp($line);

		if( $line =~ s/((?:(?:1[0-9]{2}|2(?:[0-4][0-9]|5[0-9])|[0-9]{1,2})\.){3}(?:(?:1[0-9]{2}|2(?:[0-4][0-9]|5[0-9])|[0-9]{1,2})))//){
			$dictword=$1;

			for($i=0; $i<= $#keywords; $i++){
				if($keywords[$i] eq $dictword){
					last;
				}
			}

			if($i > $#keywords){
				$keywords[$#keywords + 1]=$dictword;
				print "$dictword\n";
			}
		}
	}
}
elsif($ARGV[0] eq "get-domain-from-dnsrevenum6" || $ARGV[0] eq "get-domains-from-dnsrevenum6"){
	while($line=<STDIN>){
		chomp($line);

		if( $line =~ m/^Found:\s+((?:(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4})|(?:(?:(?:[0-9a-fA-F]{1,4}:){1,4}|:):(?:(?:(?:1[0-9]{2}|2(?:[0-4][0-9]|5[0-9])|[0-9]{1,2})\.){3}(?:(?:1[0-9]{2}|2(?:[0-4][0-9]|5[0-9])|[0-9]{1,2}))))|(?:(?:(?:(?:[0-9a-fA-F]{1,4}:)(?::[0-9a-fA-F]{1,4}){1,7})|(?:(?:[0-9a-fA-F]{1,4}:){2}(?::[0-9a-fA-F]{1,4}){1,6})|(?:(?:[0-9a-fA-F]{1,4}:){3}(?::[0-9a-fA-F]{1,4}){1,5})|(?:(?:[0-9a-fA-F]{1,4}:){4}(?::[0-9a-fA-F]{1,4}){1,4})|(?:(?:[0-9a-fA-F]{1,4}:){5}(?::[0-9a-fA-F]{1,4}){1,3})|(?:(?:[0-9a-fA-F]{1,4}:){6}(?::[0-9a-fA-F]{1,4}){1,2})|(?:[0-9a-fA-F]{1,4}:){1,7}:)|(?::(?:(?::[0-9a-fA-F]{1,4}){1,7}|:))))\s+is\s+([-a-zA-Z0-9_]+(?:\.[-a-zA-Z0-9_]+)*)/){

			$ipv6address= $1;
			$domain= $2;
			$domain=~ s/\r//g;
			$domain=lc($domain);
			print "$domain\n";
		}
	}
}
elsif($ARGV[0] eq "get-ipv6-from-dnsrevenum6" || $ARGV[0] eq "get-ipv6-addrs-from-dnsrevenum6"){
	while($line=<STDIN>){
		chomp($line);

		if( $line =~ m/^Found:\s+((?:(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4})|(?:(?:(?:[0-9a-fA-F]{1,4}:){1,4}|:):(?:(?:(?:1[0-9]{2}|2(?:[0-4][0-9]|5[0-9])|[0-9]{1,2})\.){3}(?:(?:1[0-9]{2}|2(?:[0-4][0-9]|5[0-9])|[0-9]{1,2}))))|(?:(?:(?:(?:[0-9a-fA-F]{1,4}:)(?::[0-9a-fA-F]{1,4}){1,7})|(?:(?:[0-9a-fA-F]{1,4}:){2}(?::[0-9a-fA-F]{1,4}){1,6})|(?:(?:[0-9a-fA-F]{1,4}:){3}(?::[0-9a-fA-F]{1,4}){1,5})|(?:(?:[0-9a-fA-F]{1,4}:){4}(?::[0-9a-fA-F]{1,4}){1,4})|(?:(?:[0-9a-fA-F]{1,4}:){5}(?::[0-9a-fA-F]{1,4}){1,3})|(?:(?:[0-9a-fA-F]{1,4}:){6}(?::[0-9a-fA-F]{1,4}){1,2})|(?:[0-9a-fA-F]{1,4}:){1,7}:)|(?::(?:(?::[0-9a-fA-F]{1,4}){1,7}|:))))\s+is\s+([-a-zA-Z0-9_]+(?:\.[-a-zA-Z0-9_]+)*)/){

			$ipv6address= $1;
			$domain= $2;
			$domain=~ s/\r//g;
			$domain=lc($domain);
			print "$ipv6address\n";
		}
	}
}
elsif($ARGV[0] eq "get-domain-from-script6-get" || $ARGV[0] eq "get-domains-from-script6-get" || $ARGV[0] eq "get-domain-from-script6"){
	while($line=<STDIN>){
		if($line=~ m/^#\s+(\S+)\s+\(\S+\)/){
			print "$1\n";
		}
	}
}
elsif($ARGV[0] eq "get-domain-from-zonefile" || $ARGV[0] eq "get-domain-from-zone" || $ARGV[0] eq "get-domains-from-zonefile" || $ARGV[0] eq "get-domains-from-zone"){

	$lastdomain="";

	while($line=<STDIN>){
		$line=lc($line);
		if($line=~ m/^(\S+)\s+\d+\s+in\s+ns\s+\S+$/){
			$domain= join(".", split /\./, $1);

			if($domain ne $lastdomain){
				$lastdomain=$domain;
				print "$domain\n";
			}
		}
	}
}
elsif($ARGV[0] eq "get-zonefile-domains"){

	$lastdomain="";
	$linenumber=0;
	$gotorg=0;

	while($line=<STDIN>){
		$line=lc($line);

		if(!$gotorg){
			if($line =~ m/^\s*\$origin\s+([-a-zA-Z0-9_.]+)$/){
				$org= join(".", split /\./, $1);
				$gotorg=1;
			}
			elsif($line =~ m/^\s*([-a-zA-Z0-9_.]+)\s+\d+\s+in\s+soa\s+/){
				$org= join(".", split /\./, $1);
				$gotorg=1;
			}
		}
		else{
			if($org eq "com" && $line=~ m/^\s*(\S+)\s+ns\s+\S+$/){
				$domain= $1;

				if($domain =~ m/\.$/){
					$domain= join(".", split /\./, $domain);
				}
				else{
					$domain= join(".", split /\./, $domain).".$org";
				}



				if($domain ne $lastdomain){
					$lastdomain=$domain;
					print "$domain\n";
				}
			}
			elsif($org ne "com" && $line=~ m/^(\S+)\s+\d+\s+in\s+ns\s+\S+$/){
				$domain= $1;

				if($domain =~ m/\.$/){
					$domain= join(".", split /\./, $domain);
				}
				else{
					$domain= join(".", split /\./, $domain).".$org";
				}



				if($domain ne $lastdomain){
					$lastdomain=$domain;
					print "$domain\n";
				}
			}
		}

		$linenumber++;
	}
}
elsif($ARGV[0] eq "get-zonefile-nameservers" || $ARGV[0] eq "get-zonefile-ns"){

	$lastdomain="";
	$lastns="";
	$linenumber=0;
	$gotorg=0;

	while($line=<STDIN>){
		$line=lc($line);

		if(!$gotorg){
			if($line =~ m/^\s*\$origin\s+([-a-zA-Z0-9_.]+)$/){
				$org= join(".", split /\./, $1);
				$gotorg=1;
			}
			elsif($line =~ m/^\s*([-a-zA-Z0-9_.]+)\s+\d+\s+in\s+soa\s+/){
				$org= join(".", split /\./, $1);
				$gotorg=1;
			}
		}
		else{
			if($org eq "com" && $line=~ m/^\s*(\S+)\s+ns\s+(\S+)$/){
				$domain= $1;
				$ns= $2;

				if($domain =~ m/\.$/){
					$domain= join(".", split /\./, $domain);
				}
				else{
					$domain= join(".", split /\./, $domain).".$org";
				}

				if($ns =~ m/\.$/){
					$ns = join(".", split /\./, $ns);
				}
				else{
					$ns = join(".", split /\./, $ns).".$org";
				}

				if($domain ne $lastdomain){
					$lastdomain=$domain;
					print "# $domain\n";
				}
				if($ns ne $lastns){
					$lastns= $ns;
					print "$ns\n";
				}
			}
			elsif($org ne "com" && $line=~ m/^(\S+)\s+\d+\s+in\s+ns\s+(\S+)$/){
				$domain= $1;
				$ns= $2;

				if($domain =~ m/\.$/){
					$domain= join(".", split /\./, $domain);
				}
				else{
					$domain= join(".", split /\./, $domain).".$org";
				}

				if($ns =~ m/\.$/){
					$ns = join(".", split /\./, $ns);
				}
				else{
					$ns = join(".", split /\./, $ns).".$org";
				}

				if($domain ne $lastdomain){
					$lastdomain=$domain;
					print "# $domain\n";
				}
				if($ns ne $lastns){
					$lastns= $ns;
					print "$ns\n";
				}
			}
		}

		$linenumber++;
	}
}
elsif($ARGV[0] eq "get-zonefile-mailservers" || $ARGV[0] eq "get-zonefile-mx"){

	$lastdomain="";
	$lastns="";
	$linenumber=0;
	$gotorg=0;

	while($line=<STDIN>){
		$line=lc($line);

		if(!$gotorg){
			if($line =~ m/^\s*\$origin\s+([-a-zA-Z0-9_.]+)$/){
				$org= join(".", split /\./, $1);
				$gotorg=1;
			}
			elsif($line =~ m/^\s*([-a-zA-Z0-9_.]+)\s+\d+\s+in\s+soa\s+/){
				$org= join(".", split /\./, $1);
				$gotorg=1;
			}
		}
		else{
			# XXX: The .com zonefile does not really have MX records
			if($org eq "com" && $line=~ m/^\s*(\S+)\s+mx\s+(\S+)$/){
				$domain= $1;
				$ns= $2;

				if($domain =~ m/\.$/){
					$domain= join(".", split /\./, $domain);
				}
				else{
					$domain= join(".", split /\./, $domain).".$org";
				}

				if($ns =~ m/\.$/){
					$ns = join(".", split /\./, $ns);
				}
				else{
					$ns = join(".", split /\./, $ns).".$org";
				}

				if($domain ne $lastdomain){
					$lastdomain=$domain;
					print "# $domain\n";
				}
				if($ns ne $lastns){
					$lastns= $ns;
					print "$ns\n";
				}
			}
			elsif($org ne "com" && $line=~ m/^(\S+)\s+\d+\s+in\s+mx\s+\d+\s+(\S+)$/){
				$domain= $1;
				$ns= $2;

				if($domain =~ m/\.$/){
					$domain= join(".", split /\./, $domain);
				}
				else{
					$domain= join(".", split /\./, $domain).".$org";
				}

				if($ns =~ m/\.$/){
					$ns = join(".", split /\./, $ns);
				}
				else{
					$ns = join(".", split /\./, $ns).".$org";
				}

				if($domain ne $lastdomain){
					$lastdomain=$domain;
					print "# $domain\n";
				}
				if($ns ne $lastns){
					$lastns= $ns;
					print "$ns\n";
				}
			}
		}

		$linenumber++;
	}
}
elsif($ARGV[0] eq "portscan-tcp-dual" || $ARGV[0] eq "portscan-tcp-dualstack"){
	$res   = Net::DNS::Resolver->new;

	if($#ARGV == 0){
		while($line=<STDIN>){
			chomp($line);

			if($line=~ m/^\s*#/){
				next;
			}

			if($line=~ m/^\s+$/){
				next;
			}

			$line =~ s/[^-_0-9A-Za-z.]//g;
			$line= lc($line);
			$name=$line;

			$hasv4=0;
			$hasv6=0;

			$reply = $res->query($name, "AAAA");

			if ($reply) {
				foreach $rr ($reply->answer) {
					if($rr->type eq "AAAA"){
						if(!$hasv6){
							print "# DOMAIN: $name\n";
						}

						$address=$rr->address;

						@lines=`nmap -6 -sS -Pn -n --script-args script-intensity=0 --top-ports 1000 $address`;
						print "$name#$address#";

						$resultsf=0;

						foreach $line (@lines){
							if($resultsf){
								if($line =~ m/^(\d+\/(?:tcp|udp))\s+open/){
									print "$1#";
									$resultsf=2;
								}
							}
							elsif($line =~ m/^PORT\s+STATE\s+SERVICE/){
								$resultsf=1;
							}
						}

						print "\n";

						$hasv6=1;

					}
				}
			}


			$reply = $res->query($name, "A");

			if ($reply) {

				foreach $rr ($reply->answer) {
					if($rr->type eq "A"){

						if(!$hasv6 && !$hasv4){
							print "# DOMAIN: $name\n";
						}

						$hasipv4=1;

						$address=$rr->address;

						@lines=`nmap  -sS -Pn -n --script-args script-intensity=0 --top-ports 1000 $address`;
						print "$name#$address#";

						$resultsf=0;

						foreach $line (@lines){
							if($resultsf){
								if($line =~ m/^(\d+\/(?:tcp|udp))\s+open/){
									print "$1#";
									$resultsf=2;
								}
							}
							elsif($line =~ m/^PORT\s+STATE\s+SERVICE/){
								$resultsf=1;
							}
						}

						print "\n";

						$hasv4=1;
					}
				}
			}
			
		}
	}
	else{
		$line= $ARGV[1];
		$line =~ s/[^-_0-9A-Za-z.]//g;
		$line= lc($line);
		$name=$line;

		$hasv4=0;
		$hasv6=0;

		$reply = $res->query($name, "AAAA");

		if ($reply) {
		    foreach $rr ($reply->answer) {
			if($rr->type eq "AAAA"){
				if(!$hasv6){
					print "# DOMAIN: $name\n";
				}

				$address=$rr->address;
	#			print "COMANDO: nmap -6 -Pn -n --script-args script-intensity=0 --top-ports 1000 $address\n";
				@lines=`nmap -6 -sS -Pn -n --script-args script-intensity=0 --top-ports 1000 $address`;
				print "$name#$address#";

				$resultsf=0;

				foreach $line (@lines){
					if($resultsf){
						if($line =~ m/^(\d+\/(?:tcp|udp))\s+open/){
							print "$1#";
							$resultsf=2;
						}
					}
					elsif($line =~ m/^PORT\s+STATE\s+SERVICE/){
						$resultsf=1;
					}
				}

				print "\n";

				$hasv6=1;

			}
		    }
		}


		$reply = $res->query($name, "A");

		if ($reply) {

		    	foreach $rr ($reply->answer) {
				if($rr->type eq "A"){
					if(!$hasv6 && !$hasv4){
						print "# DOMAIN: $name\n";
					}

					$hasipv4=1;

					$address=$rr->address;

					@lines=`nmap  -sS -Pn -n --script-args script-intensity=0 --top-ports 1000 $address`;
					print "$name#$address#";

					$resultsf=0;

					foreach $line (@lines){
						if($resultsf){
							if($line =~ m/^(\d+\/(?:tcp|udp))\s+open/){
								print "$1#";
								$resultsf=2;
							}
						}
						elsif($line =~ m/^PORT\s+STATE\s+SERVICE/){
							$resultsf=1;
						}
					}

					print "\n";

					$hasv4=1;

				}
			}
		}

	}
}

elsif($ARGV[0] eq "get-addresses" || $ARGV[0] eq "get-addrs"){
	$res   = Net::DNS::Resolver->new;

	if($#ARGV == 0){
		while($line=<STDIN>){
			chomp($line);

			if($line=~ m/^\s*#/){
				next;
			}

			if($line=~ m/^\s+$/){
				next;
			}

			$line =~ s/[^-_0-9A-Za-z.]//g;
			$line= lc($line);
			$name=$line;

			$hasv4=0;
			$hasv6=0;

			$reply = $res->query($name, "AAAA");

			if ($reply) {
				foreach $rr ($reply->answer) {
					if($rr->type eq "AAAA"){
						if(!$hasv6){
							print "# $name\n";
						}

						$address=$rr->address;

						print "$address\n";
						$hasv6=1;

					}
				}
			}


			$reply = $res->query($name, "A");

			if ($reply) {

				foreach $rr ($reply->answer) {
					if($rr->type eq "A"){

						if(!$hasv6 && !$hasv4){
							print "# $name\n";
						}

						$hasipv4=1;

						$address=$rr->address;
						print "$address\n";
						$hasv4=1;
					}
				}
			}
			
		}
	}
	else{
		$line= $ARGV[1];
		$line =~ s/[^-_0-9A-Za-z.]//g;
		$line= lc($line);
		$name=$line;

		$hasv4=0;
		$hasv6=0;

		$reply = $res->query($name, "AAAA");

		if ($reply) {
		    foreach $rr ($reply->answer) {
			if($rr->type eq "AAAA"){
				if(!$hasv6){
					print "# $name\n";
				}

				$address=$rr->address;
				print "$address\n";
				$hasv6=1;

			}
		    }
		}


		$reply = $res->query($name, "A");

		if ($reply) {

		    	foreach $rr ($reply->answer) {
				if($rr->type eq "A"){
					if(!$hasv6 && !$hasv4){
						print "# $name\n";
					}

					$hasipv4=1;

					$address=$rr->address;
					print "$address\n";
					$hasv4=1;

				}
			}
		}

	}
}
else{
	print "$SI6_TOOLKIT\n";
	print "$SCRIPT6\n";
	print "$USAGE\n";
}
