Initial revision
This commit is contained in:
Daniel Stenberg
261
perl/getlinks.pl.in
Executable file
261
perl/getlinks.pl.in
Executable file
@@ -0,0 +1,261 @@
|
||||
#!@PERL@
|
||||
#
|
||||
# getlinks.pl
|
||||
#
|
||||
# This script extracts all links from a HTML page, compares them to a pattern
|
||||
# entered on the command line and then downloads matching links into the
|
||||
# target dir (also specified on the command line).
|
||||
#
|
||||
# Written to use 'curl' for URL fetching, uses the source file names in the
|
||||
# target directory.
|
||||
#
|
||||
# Author: Daniel Stenberg <Daniel.Stenberg@sth.frontec.se>
|
||||
# Version: 0.1 Oct 7, 1998
|
||||
#
|
||||
# HISTORY
|
||||
#
|
||||
# 0.1 - Created now!
|
||||
#
|
||||
|
||||
$in="";
|
||||
|
||||
argv:
|
||||
if($ARGV[0] eq "-v" ) {
|
||||
$verbose = 1;
|
||||
shift @ARGV;
|
||||
goto argv;
|
||||
}
|
||||
if($ARGV[0] eq "-d" ) {
|
||||
$display = 1;
|
||||
shift @ARGV;
|
||||
goto argv;
|
||||
}
|
||||
elsif($ARGV[0] eq "-h" ) {
|
||||
$help = 1;
|
||||
shift @ARGV;
|
||||
goto argv;
|
||||
}
|
||||
|
||||
$geturl = $ARGV[0];
|
||||
$getdir = $ARGV[1];
|
||||
$getregex = $ARGV[2];
|
||||
|
||||
if(($geturl eq "") ||
|
||||
(($getdir eq "") && !$display) ||
|
||||
$help) {
|
||||
print "Usage: $0 [-hv] <full source URL> <target dir> [regex]\n",
|
||||
" Use a traling slash for directory URLs!\n",
|
||||
" Use \"quotes\" around the regex!\n",
|
||||
" -h This help text\n",
|
||||
" -d Display matches only instead of downloading\n",
|
||||
" -v Verbose mode\n";
|
||||
exit;
|
||||
}
|
||||
|
||||
# change to target directory:
|
||||
chdir $getdir ||
|
||||
die "couldn't cd into $getdir";
|
||||
|
||||
# This is necessary from where I tried this:
|
||||
#$proxy =" -x 194.237.142.41:80";
|
||||
|
||||
# linkchecker, URL will be appended to the right of this command line
|
||||
# this is the one using HEAD:
|
||||
$linkcheck = "curl -s -m 20 -I$proxy";
|
||||
|
||||
# as a second attempt, this will be used. This is not using HEAD but will
|
||||
# get the whole frigging document!
|
||||
$linkcheckfull = "curl -s -m 20 -i$proxy";
|
||||
|
||||
# htmlget, URL will be appended to the right of this command line
|
||||
$htmlget = "curl -s$proxy";
|
||||
|
||||
# urlget, URL will be appended to the right of this command line
|
||||
# this stores the file with the remote file name in the current dir
|
||||
$urlget = "curl -O -s$proxy";
|
||||
|
||||
# Parse the input URL and split it into the relevant parts:
|
||||
|
||||
sub SplitURL {
|
||||
my $inurl = $_[0];
|
||||
|
||||
if($inurl=~ /^([^:]+):\/\/([^\/]*)\/(.*)\/(.*)/ ) {
|
||||
$getprotocol = $1;
|
||||
$getserver = $2;
|
||||
$getpath = $3;
|
||||
$getdocument = $4;
|
||||
}
|
||||
elsif ($inurl=~ /^([^:]+):\/\/([^\/]*)\/(.*)/ ) {
|
||||
$getprotocol = $1;
|
||||
$getserver = $2;
|
||||
$getpath = $3;
|
||||
$getdocument = "";
|
||||
|
||||
if($getpath !~ /\//) {
|
||||
$getpath ="";
|
||||
$getdocument = $3;
|
||||
}
|
||||
|
||||
}
|
||||
elsif ($inurl=~ /^([^:]+):\/\/(.*)/ ) {
|
||||
$getprotocol = $1;
|
||||
$getserver = $2;
|
||||
$getpath = "";
|
||||
$getdocument = "";
|
||||
}
|
||||
else {
|
||||
print "Couldn't parse the specified URL, retry please!\n";
|
||||
exit;
|
||||
}
|
||||
}
|
||||
|
||||
&SplitURL($geturl);
|
||||
|
||||
#print "protocol = $getprotocol\n";
|
||||
#print "server = $getserver\n";
|
||||
#print "path = $getpath\n";
|
||||
#print "document = $getdocument\n";
|
||||
#exit;
|
||||
|
||||
if(!$usestdin) {
|
||||
open(HEADGET, "$linkcheck $geturl|") ||
|
||||
die "Couldn't get web page for some reason";
|
||||
headget:
|
||||
while(<HEADGET>) {
|
||||
# print $_;
|
||||
if($_ =~ /HTTP\/.*3\d\d /) {
|
||||
$pagemoved=1;
|
||||
}
|
||||
elsif($pagemoved &&
|
||||
($_ =~ /^Location: (.*)/)) {
|
||||
$geturl = $1;
|
||||
|
||||
&SplitURL($geturl);
|
||||
|
||||
$pagemoved++;
|
||||
last headget;
|
||||
}
|
||||
}
|
||||
close(HEADGET);
|
||||
|
||||
if($pagemoved == 1) {
|
||||
print "Page is moved but we don't know where. Did you forget the ",
|
||||
"traling slash?\n";
|
||||
exit;
|
||||
}
|
||||
|
||||
open(WEBGET, "$htmlget $geturl|") ||
|
||||
die "Couldn't get web page for some reason";
|
||||
|
||||
while(<WEBGET>) {
|
||||
$line = $_;
|
||||
push @indoc, $line;
|
||||
$line=~ s/\n//g;
|
||||
$line=~ s/\r//g;
|
||||
# print $line."\n";
|
||||
$in=$in.$line;
|
||||
}
|
||||
|
||||
close(WEBGET);
|
||||
}
|
||||
else {
|
||||
while(<STDIN>) {
|
||||
$line = $_;
|
||||
push @indoc, $line;
|
||||
$line=~ s/\n//g;
|
||||
$line=~ s/\r//g;
|
||||
$in=$in.$line;
|
||||
}
|
||||
}
|
||||
|
||||
sub GetLinks {
|
||||
my $in = $_[0];
|
||||
my @result;
|
||||
|
||||
getlinkloop:
|
||||
while($in =~ /[^<]*(<[^>]+>)/g ) {
|
||||
# we have a tag in $1
|
||||
$tag = $1;
|
||||
|
||||
if($tag =~ /^<!--/) {
|
||||
# this is a comment tag, ignore it
|
||||
}
|
||||
else {
|
||||
if($tag =~ /(src|href|background|archive) *= *(\"[^\"]\"|[^ )>]*)/i) {
|
||||
$url=$2;
|
||||
if($url =~ /^\"(.*)\"$/) {
|
||||
# this was a "string" now $1 has removed the quotes:
|
||||
$url=$1;
|
||||
}
|
||||
|
||||
|
||||
$url =~ s/([^\#]*)\#.*/$1/g;
|
||||
|
||||
if($url eq "") {
|
||||
# if the link was nothing than a #-link it may now have
|
||||
# been emptied completely so then we skip the rest
|
||||
next getlinkloop;
|
||||
}
|
||||
|
||||
if($done{$url}) {
|
||||
# if this url already is done, do next
|
||||
$done{$url}++;
|
||||
next getlinkloop;
|
||||
}
|
||||
|
||||
$done{$url} = 1; # this is "done"
|
||||
|
||||
push @result, $url;
|
||||
if($tag =~ /< *([^ ]+)/) {
|
||||
# print "TAG: $1\n";
|
||||
$tagtype{$url}=$1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return @result;
|
||||
}
|
||||
|
||||
@links = &GetLinks($in);
|
||||
|
||||
linkloop:
|
||||
for(@links) {
|
||||
$url = $_;
|
||||
|
||||
if($url =~ /^([^:]+):/) {
|
||||
$link = $url;
|
||||
}
|
||||
else {
|
||||
# this is an absolute link on the same server:
|
||||
if($url =~ /^\//) {
|
||||
# from root
|
||||
$link = "$getprotocol://$getserver$url";
|
||||
}
|
||||
else {
|
||||
# from the scanned page's dir
|
||||
$nyurl=$url;
|
||||
|
||||
if(length($getpath) &&
|
||||
($getpath !~ /\/$/) &&
|
||||
($nyurl !~ /^\//)) {
|
||||
# lacks ending slash, add one to the document part:
|
||||
$nyurl = "/".$nyurl;
|
||||
}
|
||||
$link = "$getprotocol://$getserver/$getpath$nyurl";
|
||||
}
|
||||
}
|
||||
|
||||
if($link =~ /$getregex/) {
|
||||
if($display) {
|
||||
print "$link\n";
|
||||
}
|
||||
else {
|
||||
if($verbose) {
|
||||
print "Gets $link\n";
|
||||
}
|
||||
print `$urlget $link`;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user