對 wget 這個 tool 不熟，平常使用 wget 下載一些資料時，可以輕易地使用 --referer 來偽造 HTTP Header 資料，因此能夠通過對方 Server 檢查
wget --referer="REFERER_URL" "TARGET_URL"
然而，上述的 REFERER_URL 和 TARGET_URL 都是固定的位置，如果是會根據 session / cookie 的而改變的話，不曉得還有沒有辦法？對我而言，寫 PHP 比去看 manpage 來得快 XD 所以我就寫成 PHP 囉！或許 wget 也有更方便的下法吧，改天再努力看 manpage
程式碼：
<?php
$output_file = 'result.file'; // 儲存結果
$cookie_file = 'cookie.tmp'; // cookie file
$source_url = 'SOURCE_URL'; // 之後會變成 REFERER_URL
$pattern = '/class="download" href="(.*?)"/'; // 此為一個範例, 用來撈 TARGET_URL
$ch = curl_init();
curl_setopt( $ch , CURLOPT_URL, $source_url );
curl_setopt( $ch , CURLOPT_COOKIEFILE , $cookie_file );
curl_setopt( $ch , CURLOPT_COOKIEJAR , $cookie_file );
curl_setopt( $ch , CURLOPT_RETURNTRANSFER , true );
$result = curl_exec( $ch );
if( preg_match_all( $pattern , $result , $match ) )
{
if( isset( $match[1][1] ) )
{
$target_url = $match[1][1]; // 請依 pattern 決定
$referer_url = $source_url;
curl_setopt( $ch , CURLOPT_URL, $target_url );
curl_setopt( $ch , CURLOPT_REFERER , $referer_url );
curl_setopt( $ch , CURLOPT_COOKIEFILE , $cookie_file );
curl_setopt( $ch , CURLOPT_COOKIEJAR , $cookie_file );
//curl_setopt( $ch , CURLOPT_RETURNTRANSFER , true );
$fp = fopen ( $output_file, 'wb' );
curl_setopt( $ch , CURLOPT_FILE , $fp );
echo "GO...\n";
curl_exec( $ch );
echo "Finish..\n";
fclose( $fp );
}
}
curl_close( $ch );
?>
以上是要從 SOURCE_URL 上頭, 找到下載位置(target_url), 然而, 那個位置卻每次都不一樣, 最重要的是跟 session 有關係並且下載 target_url 時還必須奉上 cookie 資訊, 所以, 先收集一下 cookie 囉！(上述程式並不謹慎, 例如儲存結果的檔案有可能開檔失敗)
後記，無聊又改寫成 tool mode：
<?php
$shortopt = array();
$shortopt['h'] = array(
'value' => '' ,
'text' => '-h, help' );
$shortopt['c:'] = array(
'value' => '' ,
'text' => "-c '/tmp/cookie_file' , tmp file for cookie" );
$shortopt['o:'] = array(
'value' => '' ,
'text' => "-o '/tmp/output_file' , path for result file. default use stdout" );
$shortopt['u:'] = array(
'value' => NULL ,
'text' => "-u 'http://www.google.com' , source url" );
$shortopt['e:'] = array(
'value' => NULL ,
'text' => "-e '/class=\"normal-down\" href=\"(.*?)\"/is' , regexp pattern for extract the target url" );
$shortopt['m:'] = array(
'value' => '' ,
'text' => "-m '1,1' , choose the result matched to be used. e.g. use the match[5][2] is '5,2'" );
$shortopt['d'] = array(
'value' => 'true' ,
'text' => "-d , disable test mode for showing the target matched by regexp pattern" );
// check function
if( !function_exists( 'getopt' ) )
{
echo "'getopt' is not supported in current PHP version.\n";
exit;
}
// help menu
$shortopt_list = '';
$shottopt_help = '';
foreach( $shortopt as $k => $v )
{
$shortopt_list .= $k;
$shottopt_help .= "\t".$v['text']."\n";
}
// start to parse...
$parse_arg = getopt( $shortopt_list );
// show help
if( isset( $parse_arg['h'] ) )
{
echo "Usage> php ".$argv[0]." -h\n";
echo $shottopt_help;
exit;
}
// set the value
foreach( $parse_arg as $k => $v )
{
if( isset( $shortopt[$k] ) )
$shortopt[$k]['value'] = !strcasecmp( $shortopt[$k]['value'] , 'false' ) ? true : false ;
else if( isset( $shortopt[$k.':'] ) )
$shortopt[$k.':']['value'] = $v;
}
// check value (cannot be NULL)
$check_out = '';
foreach( $shortopt as $k => $v )
if( !isset( $v['value'] ) )
$check_out .= "\t".$v['text']."\n";
if( !empty( $check_out ) )
{
echo "Usage> php ".$argv[0]." -h\n";
echo "Must Set:\n$check_out\n";
exit;
}
$cookie_file = !empty( $shortopt['c:']['value'] ) ? $shortopt['c:']['value'] : NULL ;
$source_url = $shortopt['u:']['value'];
$output_file = !empty( $shortopt['o:']['value'] ) ? $shortopt['o:']['value'] : NULL ;
$regexp_pattern = $shortopt['e:']['value'];
if( !empty( $shortopt['m:']['value'] ) )
$shortopt['m:']['value'] = trim( $shortopt['m:']['value'] );
$choose_match = !empty( $shortopt['m:']['value'] ) ? explode( ',' , $shortopt['m:']['value'] ) : NULL;
$test_mode = empty( $choose_match ) || $shortopt['d']['value'];
$ch = curl_init();
curl_setopt( $ch , CURLOPT_URL, $source_url );
if( !empty( $cookie_file ) )
{
curl_setopt( $ch , CURLOPT_COOKIEFILE , $cookie_file );
curl_setopt( $ch , CURLOPT_COOKIEJAR , $cookie_file );
}
curl_setopt( $ch , CURLOPT_RETURNTRANSFER , true );
$result = curl_exec( $ch );
if( preg_match_all( $regexp_pattern , $result , $matches ) )
{
$target_url = getTargetURL( $matches , $choose_match );
if( $test_mode || empty( $target_url ) )
{
echo "Matched Target URL: \n";
print_r( $matches );
echo "Choose option(Cannot be empty):".$shortopt['m:']['value']."\n";
echo "Target(Cannot be empty):$target_url\n";
}
else
{
curl_setopt( $ch , CURLOPT_URL, $target_url );
curl_setopt( $ch , CURLOPT_REFERER , $source_url );
if( !empty( $cookie_file ) )
{
curl_setopt( $ch , CURLOPT_COOKIEFILE , $cookie_file );
curl_setopt( $ch , CURLOPT_COOKIEJAR , $cookie_file );
}
if( !empty( $output_file ) )
{
echo "Target URL:$target_url\n";
echo "Referer URL:$source_url\n";
if( ( $fp = fopen ( $output_file , 'wb' ) ) == NULL )
{
echo "ERROR: Cannot open the output file to write:$output_file\n";
exit;
}
curl_setopt( $ch , CURLOPT_FILE , $fp );
echo "Begin...\n";
curl_exec( $ch );
echo "...Finish\n";
fclose( $fp );
}
else
{
curl_exec( $ch );
}
}
}
curl_close( $ch );
exit;
function getTargetURL( $matches , $choose )
{
if( !isset( $matches ) )
return NULL;
if( is_array( $matches ) && is_array( $choose ) && count( $choose ) > 0 )
{
$index = array_shift( $choose );
if( isset( $matches[ $index ] ) )
return getTargetURL( $matches[ $index ] , $choose );
return NULL;
}
if( !is_array( $matches ) )
return $matches;
else if( isset( $matches[ $choose ] ) )
return $matches[ $choose ];
return NULL;
}
?>
用法：
單純以抓 Yahoo! New 為例
尚未指定 -m
# php my_wget.php -u 'http://tw.yahoo.com' -e '/<h3><a href="([^"]+)" title="([^"]+)"/is'
Matched Target URL:
Array
(
[0] => Array
(
[0] => <h3><a href="news/a/h1/t/*http://tw.news.yahoo.com/article/url/d/a/100628/5/289yr.html" title="莫拉克風災 學者：無關暖化"
[1] => <h3><a href="news/a/h2/t/*http://tw.news.yahoo.com/article/url/d/a/100628/69/289tr.html" title="立院藏七寶 總價數億元"
)
[1] => Array
(
[0] => news/a/h1/t/*http://tw.news.yahoo.com/article/url/d/a/100628/5/289yr.html
[1] => news/a/h2/t/*http://tw.news.yahoo.com/article/url/d/a/100628/69/289tr.html
)
[2] => Array
(
[0] => 莫拉克風災 學者：無關暖化
[1] => 立院藏七寶 總價數億元
)
)
Choose option(Cannot be empty):
Target(Cannot be empty):
指定 -m '1,1'
# php my_wget.php -u 'http://tw.yahoo.com' -e '/<h3><a
href="([^"]+)" title="([^"]+)"/is' -m '1,1'
Matched Target URL:
Array
(
[0] => Array
(
[0] => <h3><a href="news/a/h1/t/*http://tw.news.yahoo.com/article/url/d/a/100628/5/289yr.html" title="莫拉克風災 學者：無關暖化"
[1] => <h3><a href="news/a/h2/t/*http://tw.news.yahoo.com/article/url/d/a/100628/69/289tr.html" title="立院藏七寶 總價數億元"
)
[1] => Array
(
[0] => news/a/h1/t/*http://tw.news.yahoo.com/article/url/d/a/100628/5/289yr.html
[1] => news/a/h2/t/*http://tw.news.yahoo.com/article/url/d/a/100628/69/289tr.html
)
[2] => Array
(
[0] => 莫拉克風災 學者：無關暖化
[1] => 立院藏七寶 總價數億元
)
)
Choose option(Cannot be empty):1,1
Target(Cannot be empty):news/a/h2/t/*http://tw.news.yahoo.com/article/url/d/a/100628/69/289tr.html
正式要下載請記得加 -d (disable test) , 但此例不適用, 因為抓出來的 url 並不完整, 開頭只是 "news/a/h2/t/*....."
# php my_wget.php -u 'http://tw.yahoo.com' -e '/<h3><a
href="([^"]+)" title="([^"]+)"/is' -m '1,1' -d
輸出到檔案
# php my_wget.php -u
'http://tw.yahoo.com' -e '/<h3><a
href="([^"]+)" title="([^"]+)"/is' -m '1,1' -d -o '/tmp/output'
需要 cookie
# php my_wget.php -u
'http://tw.yahoo.com' -e '/<h3><a
href="([^"]+)" title="([^"]+)"/is' -m '1,1' -d -o '/tmp/output' -c '/tmp/cookie'
