package webCrawler.Spider;

import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Scanner;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import webCrawler.Index.BuildIndex;

/**
 * @author lannooo
 */

public class Spider {
  ArrayList URLs;
  private String startURL;
  private int digLevel;

  /**
   * @param startURL 爬蟲的起始URL
   * @param digLevel 爬取深度
   */
  public Spider(String startURL, int digLevel){
    this.startURL = startURL;
    this.digLevel = digLevel;
    this.URLs = new ArrayList<>();
  }

  /**
   * @param level 當(dāng)前爬取的深度剩余
   * @param arrayList 需要進(jìn)行下一輪爬去的URL集
   * @return 從一格url集爬取到的新的URL集
   * @throws IOException
   */
  public ArrayList getLevelURLs(int level, ArrayList arrayList) 
      throws IOException{
    ArrayList total = null;
    if(level>0){      
      total = new ArrayList<>();
      for(String url: arrayList){
        /*對(duì)于每個(gè)arrayList中的URL，首先解析其網(wǎng)頁內(nèi)容，并獲得里面所有URL項(xiàng)*/
        for(String each: getBareLinks(url)){
          total.add(each);
        }
      }
      /*用HashSet這個(gè)容器將total里面重復(fù)項(xiàng)刪除*/
      HashSet hashSet = new HashSet<>(total);
      total = new ArrayList<>(hashSet);
    }
    return total;
  }

  /**
   * 從startURL開始，爬取所有相關(guān)URLs
   * @throws IOException
   */
  public void getAll() throws IOException{
    ArrayList newURLs;
    ArrayList currentURLs = new ArrayList<>();
    /*把startURL加入currentURLs這個(gè)列表中，從這個(gè)url開始爬*/
    currentURLs.add(startURL);
    for(int i=digLevel; i>0; i--){
      /*
       * 對(duì)于每一層，都要獲取一次由這個(gè)url引申出去的url集
       * 然后把當(dāng)前集的已經(jīng)爬去過的url加入到總的URL集中
       * 最后newURLs作為新的需要進(jìn)行深度爬取的集進(jìn)入下一輪循環(huán)
       */
      System.out.println("Dig into level: " + (digLevel-i+1));
      newURLs = getLevelURLs(i, currentURLs);
      for(String each: currentURLs){
        URLs.add(each);
      }
      currentURLs = newURLs;
    }
    for(String each:currentURLs){
      URLs.add(each);
    }
    HashSet hashSet = new HashSet<>(URLs);
    URLs = new ArrayList<>(hashSet);
  }

  /**
   * @param path 保存索引的路徑
   * @throws IOException
   */
  public void storeURLsAndInfo(String path) throws IOException{
    BuildIndex build = new BuildIndex(path);
    /* 把URLs中的所有url進(jìn)行實(shí)際網(wǎng)頁標(biāo)題的爬取*/
    for(String each:URLs){
      String text = getLinkText(each);
      if(text!=null){
        build.addField("url", each);
        build.addField("text", text);
        /*將這一個(gè)entry加入索引中*/
        build.pushIndex();
      }
    }
    build.close();
  }

  /**
   * @param url 需要獲取網(wǎng)頁標(biāo)題的url
   * @return 標(biāo)題內(nèi)容
   * @throws IOException
   */
  public String getLinkText(String url) throws IOException{
    Document document = null;
    try {
      /*用Jsoup進(jìn)行連接，設(shè)置超時(shí)時(shí)間為3秒*/
      document = Jsoup.connect(url).timeout(3000).get();
    } catch (Exception e) {
      System.out.println("[TIMEOUT]Get title of url:"+url);
      return null;
    }
    String title = document.title();
    return title;
  }


  /**
   * @param url 進(jìn)行內(nèi)容解析的url
   * @return 返回該url的網(wǎng)頁內(nèi)容內(nèi)的所有urls列表
   * @throws IOException
   */
  public ArrayList getBareLinks(String url) throws IOException{
    ArrayList linksList = new ArrayList<>();
    Document document;

    try {
      document = Jsoup.connect(url).timeout(2000).get();
    } catch (Exception e) {
      return linksList;
    }
    /*獲取標(biāo)簽理的所有帶href屬性的標(biāo)簽*/
    Elements links = document.select("body").select("a[href]");

    for(Element link: links){
      /*從每一個(gè)解析得到的標(biāo)簽中提取url，并去除錨點(diǎn)*/
      String href = link.attr("abs:href").replaceAll("#", "");
      /*只添加含有zju.edu.cn字符的url，去除末尾的'/'*/
      if(href.contains("zju.edu.cn")){
        if (href.endsWith("/")){
          href = href.substring(0, href.length()-1);
        }
        linksList.add(href);
      }
    }
    HashSet hashSet = new HashSet<>(linksList);
    ArrayList arrayList = new ArrayList<>(hashSet);

    return arrayList;
  }

  public static void main(String[] args) {
    Scanner in = new Scanner(System.in);
    System.out.println("Enter url:");
    String url = in.nextLine().trim();
    while(!url.startsWith("http://")){
      System.out.println("http:// is needed!");
      System.out.println("Enter url:");
      url = in.nextLine().trim();
    }
    System.out.println("Enter depth to dig more urls[<=3 recommended]：");
    int depth = in.nextInt();
    Spider spider = new Spider(url, depth);
    System.out.println("Enter path you want to save[default=d:/index-spider]:");
    String path = in.nextLine().trim();
    if(path.length()==0){
      path = "d:/index-spider";
    }
    try {
      System.out.println("Start fetching...");
      spider.getAll();
      System.out.println("Urls got success!");
      spider.storeURLsAndInfo(path);
      System.out.println("Stored success!");
    } catch (IOException e) {
      e.printStackTrace();
    }
  }

 
}

package webCrawler.Index;

import java.io.*;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Version;
import org.wltea.analyzer.lucene.IKAnalyzer;

/**
 * @author lannooo
 *
 */
public class BuildIndex {
  private File file;
  private Directory directory;
  private IndexWriter indexWriter;
  private IndexWriterConfig config;
  private Analyzer analyzer;
  private Document document;

  /**
   * @param path 建立索引的路徑
   */
  public BuildIndex(String path) {
    try {
      file = new File(path);
      directory = FSDirectory.open(file);
      document = new Document();
      analyzer = new IKAnalyzer();    /*中文分詞工具類*/
      config = new IndexWriterConfig(Version.LUCENE_4_10_0, analyzer);
      indexWriter = new IndexWriter(directory, config);      

    } catch (Exception e) {
      e.printStackTrace();
    }
  }

  /**
   * @param fieldName 加入到document中的新的一項(xiàng)的名稱
   * @param fieldText 新的一項(xiàng)的內(nèi)容
   */
  public void addField(String fieldName, String fieldText){
    try{
      Field field = new TextField(fieldName, fieldText, Field.Store.YES);
      document.add(field);
    }catch (Exception e) {
      e.printStackTrace();
    }
  }

  /**
   * 將document加入到索引中
   */
  public void pushIndex(){
    try {
      indexWriter.addDocument(document);
      document = new Document();
    } catch (Exception e) {
      e.printStackTrace();
    }
  }

  /**
   * 加入完整的一個(gè)document并保存到索引中
   * @param url 加入的url地址
   * @param text url對(duì)應(yīng)的文本
   */
  public void addOneIndex(String url, String text){
    this.addField("url", url);
    this.addField("text", text);
    this.pushIndex();
  }

  /**
   * 關(guān)閉索引寫入
   */
  public void close(){
    try {
      indexWriter.close();
    } catch (Exception e) {
      e.printStackTrace();
    }
  }

}

package webCrawler.Index;

import java.io.File;
import java.util.Scanner;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.queryparser.classic.QueryParser;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.FSDirectory;
import org.wltea.analyzer.lucene.IKAnalyzer;

/**
 * @author lannooo
 *
 */
public class SearchIndex {
  private IndexSearcher indexSearcher;
  private Analyzer analyzer;
  private QueryParser parser;
  private Query query;
  private TopDocs hits;
  private DirectoryReader reader;

  /**
   * @param path 進(jìn)行索引搜索的路徑
   */
  public SearchIndex(String path){
    try {
      reader = DirectoryReader.open(FSDirectory.open(new File(path)));
      indexSearcher = new IndexSearcher(reader);
      analyzer = new IKAnalyzer();
    } catch (Exception e) {
      e.printStackTrace();
    }
  }

  /**
   * @param fieldName 搜索的域名稱
   * @param text 搜索的內(nèi)容
   * @param matchNumber 最大匹配項(xiàng)數(shù)
   * @return 搜索到的最大匹配數(shù)
   */
  public int search(String fieldName, String text, int matchNumber){
    try {
      parser = new QueryParser(fieldName, analyzer);
      query = parser.parse(text);
      hits = indexSearcher.search(query, matchNumber);

      return hits.totalHits;
    } catch (Exception e) {
      e.printStackTrace();
    }
    return -1;
  }
  /**
   * 打印所有的匹配項(xiàng)
   */
  public void printHits(){
    try{
      System.out.println("Total hits number:"+hits.totalHits);
      for(ScoreDoc doc: hits.scoreDocs){
        Document document = indexSearcher.doc(doc.doc);
        System.out.println(document.get("url"));
        System.out.println(document.get("text"));
      }
      reader.close();
    }catch (Exception e) {
      e.printStackTrace();
    }
  }
  public static void main(String[] args) {
    /*輸入關(guān)鍵詞*/
    Scanner in = new Scanner(System.in);
    System.out.println("Enter path of the index:");
    String path = in.nextLine().trim();
    while(path.length()==0){
      System.out.println("Enter path of the index:");
      path = in.nextLine().trim();
    }

    System.out.println("Enter max hit number:");
    int max = in.nextInt();
    while(max<0){
      System.out.println("Enter max hit number:");
      max = in.nextInt();
    }
    in.nextLine();
    System.out.print("Search>>> ");
    String text = in.nextLine().trim();
    /*循環(huán)讀入用戶的關(guān)鍵詞，如果是q則退出，長度為0也退出*/
    while(!text.equals("q")){
      if(text.length()>0){
        SearchIndex search = new SearchIndex(path);
        int hits = search.search("text", text, max);
        if(hits!=-1){
          search.printHits();
        }
      }
      System.out.print("Search>>> ");
      text = in.nextLine().trim();
    }
  }
}

package webCrawler.UI;

import java.util.Scanner;

import webCrawler.Index.SearchIndex;

/**
 * @author lannooo
 *
 */
public class UI {
  public static void main(String[] args) {
    /*輸入關(guān)鍵詞*/
    Scanner in = new Scanner(System.in);
    System.out.print("Search>>> ");
    String text = in.nextLine().trim();
    /*對(duì)于用戶的關(guān)鍵詞，如果是q則退出，長度為0也退出*/
    while(!text.equals("q") && text.length()>0){
      SearchIndex search = new SearchIndex("d:/index-spider2");
      int hits = search.search("text", text, 20);
      if(hits!=-1){
        search.printHits();
      }
      System.out.print("Search>>> ");
      text = in.nextLine().trim();
    }
  }
}