待下載的網(wǎng)頁(yè)地址放在$urls數(shù)組中,按指定的并發(fā)數(shù)多進(jìn)程下載網(wǎng)頁(yè),下載的網(wǎng)頁(yè)保存在本地硬盤,下載的網(wǎng)頁(yè)大小通過(guò)linux消息隊(duì)列發(fā)送給父進(jìn)程累加,全部網(wǎng)頁(yè)下載完成后,父進(jìn)程顯示下載的網(wǎng)頁(yè)數(shù)、字節(jié)數(shù)。代碼如下。
創(chuàng)新互聯(lián)建站主要從事網(wǎng)站制作、成都做網(wǎng)站、網(wǎng)頁(yè)設(shè)計(jì)、企業(yè)做網(wǎng)站、公司建網(wǎng)站等業(yè)務(wù)。立足成都服務(wù)佳木斯,十年網(wǎng)站建設(shè)經(jīng)驗(yàn),價(jià)格優(yōu)惠、服務(wù)專業(yè),歡迎來(lái)電咨詢建站服務(wù):18980820575
//$urls數(shù)組用于保存要下載的網(wǎng)址,實(shí)際應(yīng)用中一般從文件或數(shù)據(jù)庫(kù)中讀取網(wǎng)址保存到$urls中。 $urls = array('http://www.qq.com','http://www.sohu.com','http://www.sina.com.cn',....); $urls_num = count($urls);//數(shù)組大小,也是網(wǎng)址數(shù)量 $msg_file = "/tmp/download_msgqueue.txt";//下面3行創(chuàng)建linux消息隊(duì)列下,該文件須先創(chuàng)建好 $msg_queuekey = ftok($msg_file,'R');//touch /tmp/download_msgqueue.txt $msg_queue = msg_get_queue($msg_queuekey, 0666); $maxtasknum = 5;//設(shè)定并發(fā)進(jìn)程數(shù) $ct = 0;//$urls數(shù)組用的計(jì)數(shù)器 $cttask = 0;//并發(fā)進(jìn)程計(jì)數(shù)器 $pids = array();//保存進(jìn)程的數(shù)組 $total_bytes = 0;//下載網(wǎng)頁(yè)的字節(jié)數(shù) while ($ct<$urls_num) {//循環(huán)抓取$urls數(shù)組中指定的網(wǎng)頁(yè) while ($cttask<$maxtasknum && $ctproc<$urls_num) {//fork出指定的并發(fā)數(shù)進(jìn)程 $pids[$ct] = pcntl_fork(); if ($pids[$ct]==-1) { echo "create subproc fail.\n"; exit(0); } elseif ($pids[$ct]>0) {//父進(jìn)程 } elseif ($pids[$ct]==0) {//子進(jìn)程 download($urls[$ct], $msg_queue); exit(0); } $cttask++; $ct++; } $tmppid = pcntl_waitpid(0, $status);//等待子進(jìn)程結(jié)束 foreach($pids as $key => $pid) { if($tmppid == $pid){ unset($pids[$key]); $cttask--;//子進(jìn)程結(jié)束后,并發(fā)進(jìn)程計(jì)數(shù)器減1 } } do {//從消息隊(duì)列出取出每個(gè)網(wǎng)頁(yè)的大小,計(jì)算下載的字節(jié)數(shù)。如果要下載的網(wǎng)頁(yè)很多,需要把此段代碼放到下載網(wǎng)頁(yè)的循環(huán)中,否則可能會(huì)出現(xiàn)隊(duì)列滿的情況。 msg_receive($msg_queue, 0, $message_type, 16, $message, true, MSG_IPC_NOWAIT); //echo "[".$message."]\n"; $total_bytes += $message; $a = msg_stat_queue($msg_queue); if($a['msg_qnum'] == 0){//這種方式退出比$ct==$urls_num好,因?yàn)槿绻鹒ork==-1,就不會(huì)有$urls_num個(gè)消息,程序會(huì)永遠(yuǎn)等待消息。 break; } } while(true); } while ($cttask > 0) {//等待最后$cttask個(gè)子進(jìn)程結(jié)束 $tmppid = pcntl_waitpid(0,$status); foreach($pids as $key => $pid) { if($tmppid == $pid){ unset($pids[$key]); $cttask--; } } } do {//取得最后$cttask個(gè)子進(jìn)程的消息 msg_receive($msg_queue, 0, $message_type, 16, $message, true, MSG_IPC_NOWAIT); //echo "[".$message."]\n"; $total_bytes += $message; $a = msg_stat_queue($msg_queue); if($a['msg_qnum'] == 0){ break; } } while(true); msg_remove_queue($msg_queue);//刪除消息隊(duì)列 echo "\nDone. download: ".$urls_num." pages,total: ".round($total_bytes/1024,3)." KB \n"; exit(0); function download($url, $msg_queue) {//下載指定網(wǎng)頁(yè),把內(nèi)容保存在本地硬盤,并下載內(nèi)容的長(zhǎng)度放入消息隊(duì)列中 $dirname = "/tmp/donwload/";//保存下載網(wǎng)頁(yè)的目錄,要事先創(chuàng)建好 $content = file_get_contents($url); if ($content === false) { $content = 0; } $url_parts = parse_url($url); $fname = $dirname.$url_parts['host']; $ret = file_put_contents($fname, $content); msg_send($msg_queue, 1, strlen($content)); } ?>
參考資料:
PHP實(shí)現(xiàn)進(jìn)程間通信:消息隊(duì)列 https://www.douban.com/note/245520545/