php写的关于静态页面的蜘蛛爬行记录的代码

php中文网

发布： 2016-07-25 09:02:54

原创

1134人浏览过

$useragent = addslashes(strtolower($_server['http_user_agent']));
if (strpos($useragent, 'googlebot')!== false){$bot = 'google';}
elseif (strpos($useragent,'mediapartners-google') !== false){$bot = 'google adsense';}
elseif (strpos($useragent,'baiduspider') !== false){$bot = 'baidu';}
elseif (strpos($useragent,'sogou spider') !== false){$bot = 'sogou';}
elseif (strpos($useragent,'sogou web') !== false){$bot = 'sogou web';}
elseif (strpos($useragent,'sosospider') !== false){$bot = 'soso';}
elseif (strpos($useragent,'yahoo') !== false){$bot = 'yahoo';}
elseif (strpos($useragent,'msn') !== false){$bot = 'msn';}
elseif (strpos($useragent,'msnbot') !== false){$bot = 'msnbot';}
elseif (strpos($useragent,'sohu') !== false){$bot = 'sohu';}
elseif (strpos($useragent,'yodaobot') !== false){$bot = 'yodao';}
elseif (strpos($useragent,'twiceler') !== false){$bot = 'twiceler';}
elseif (strpos($useragent,'ia_archiver') !== false){$bot = 'alexa_';}
elseif (strpos($useragent,'iaarchiver') !== false){$bot = 'alexa';}
elseif (strpos($useragent,'slurp') !== false){$bot = '雅虎';}
elseif (strpos($useragent,'bot') !== false){$bot = '其它蜘蛛';}
if(isset($bot)){
$fp = @fopen('bot.txt','a');
fwrite($fp,date('y-m-d h:i:s')."\t".$_server["remote_addr"]."\t".$bot."\t".'http://'.$_server['server_name'].$_server["http_x_rewrite_url"]."\r\n");
fclose($fp);
}
$file=".".$_server[http_x_rewrite_url];
$f_head=substr($file,-5);
if($f_head==".html")
{
if(file_exists($file))
{
echo file_get_contents($file);
}else
{
header('http/1.1 404 not found');
header("status: 404 not found");
echo "该页面无法找到";
}
}
else
{
header('HTTP/1.1 404 Not Found');
header("status: 404 Not Found");
echo "该页面无法找到";
}
?>

复制代码

伪静态文件内容：

[ISAPI_Rewrite]

代码小浣熊
代码小浣熊是基于商汤大语言模型的软件智能研发助手，覆盖软件需求分析、架构设计、代码编写、软件测试等环节

51

查看详情
# 3600 = 1 hour
CacheClockRate 3600
RepeatLimit 32
# Protect httpd.ini and httpd.parse.errors files
# from accessing through HTTP
RewriteRule /index.html /index.php
RewriteRule ^/article/(.*) /bot.php [L]
RewriteRule ^/list/(.*) /bot.php [L]