How to make nutch run in eclipse ?

﹏ヽ暗。殇╰゛Y 2022-04-13 10:24 208阅读 0赞

<?xml version="1.0"?>
    <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
    
    
    
    <configuration>
    
    <property>
      <name>http.agent.name</name>
      <value>mynutch</value>
    </property>
    
    
    <property>
      <name>http.robots.agents</name>
      <value>*</value>
      <description>The agent strings we'll look for in robots.txt files,
      comma-separated, in decreasing order of precedence. You should
      put the value of http.agent.name as the first agent name, and keep the
      default * at the end of the list. E.g.: BlurflDev,Blurfl,*
      </description>
    </property>
    
    <property>
      <name>plugin.folders</name>
      <value>./src/plugin</value>
      <description>Directories where nutch plugins are located.  Each
      element may be a relative or absolute path.  If absolute, it is used
      as is.  If relative, it is searched for on the classpath.</description>
    </property>
    
    </configuration>

\[b\]\[color=green\]\[size=large\]下面简单介绍下，在各个类里运行，需要做的一些改动，首先运行nutch，是基于Hadoop的local模式调试的，所以，你得改变下hadoop的权限，否则在运行过程中，会报错。散仙在这里提供一个简单的方法，拷贝hadoop的FileUtils类进行eclipse中，修改它的权限校验即可，如果你是在linux上运行，就不需要考虑这个问题了。  
  
在开始调试之前，你需要在项目的根目录下建一个urls文件夹，并新建一个种子文件放入你要抓取的网址。  
  
在Injector类里面，run方法里，改成  
\[/size\]\[/color\]\[/b\]

public int run(String[] args) throws Exception {
       
    //    if (args.length < 2) {
       
    //      System.err.println("Usage: Injector <crawldb> <url_dir>");
    //      return -1;
    //    }
    	  args=new String[]{"mydir","urls"};//urls
        try {
       
          inject(new Path(args[0]), new Path(args[1]));
          return 0;
        } catch (Exception e) {
       
          LOG.error("Injector: " + StringUtils.stringifyException(e));
          return -1;
        }
      }

\[color=olive\]\[size=large\]在Generator里面的run方法改成\[/size\]\[/color\]

public int run(String[] args) throws Exception {
       
    //    if (args.length < 2) {
       
    //      System.out
    //          .println("Usage: Generator <crawldb> <segments_dir> [-force] [-topN N] [-numFetchers numFetchers] [-adddays numDays] [-noFilter] [-noNorm][-maxNumSegments num]");
    //      return -1;
    //    }
    
    	  args=new String[]{"mydir","myseg","6","7",""};
    
        Path dbDir = new Path(args[0]);
        Path segmentsDir = new Path(args[1]);
        long curTime = System.currentTimeMillis();
        long topN = Long.MAX_VALUE;
        int numFetchers = -1;
        boolean filter = true;
        boolean norm = true;
        boolean force = false;
        int maxNumSegments = 1;
    
        for (int i = 2; i < args.length; i++) {
       
          if ("-topN".equals(args[i])) {
       
            topN = Long.parseLong(args[i + 1]);
            i++;
          } else if ("-numFetchers".equals(args[i])) {
       
            numFetchers = Integer.parseInt(args[i + 1]);
            i++;
          } else if ("-adddays".equals(args[i])) {
       
            long numDays = Integer.parseInt(args[i + 1]);
            curTime += numDays * 1000L * 60 * 60 * 24;
          } else if ("-noFilter".equals(args[i])) {
       
            filter = false;
          } else if ("-noNorm".equals(args[i])) {
       
            norm = false;
          } else if ("-force".equals(args[i])) {
       
            force = true;
          } else if ("-maxNumSegments".equals(args[i])) {
       
            maxNumSegments = Integer.parseInt(args[i + 1]);
          }
    
        }
    
        try {
       
          Path[] segs = generate(dbDir, segmentsDir, numFetchers, topN, curTime, filter,
              norm, force, maxNumSegments);
          if (segs == null) return -1;
        } catch (Exception e) {
       
          LOG.error("Generator: " + StringUtils.stringifyException(e));
          return -1;
        }
        return 0;
      }

\[color=green\]\[size=large\]在Fetcher的run方法里面改动：\[/size\]\[/color\]

public int run(String[] args) throws Exception {
       
    
        String usage = "Usage: Fetcher <segment> [-threads n]";
    
         args=new String[]{"D:\\20140520nutchplugin\\apache-nutch-1.8\\myseg\\20140520120541","4"};
    //    if (args.length < 1) {
       
    //      System.err.println(usage);
    //      return -1;
    //    }
    
        Path segment = new Path(args[0]);
    
        int threads = getConf().getInt("fetcher.threads.fetch", 10);
        boolean parsing = false;
    
        for (int i = 1; i < args.length; i++) {       // parse command line
          if (args[i].equals("-threads")) {           // found -threads option
            threads =  Integer.parseInt(args[++i]);
          }
        }
    
        getConf().setInt("fetcher.threads.fetch", threads);
    
        try {
       
          fetch(segment, threads);
          return 0;
        } catch (Exception e) {
       
          LOG.error("Fetcher: " + StringUtils.stringifyException(e));
          return -1;
        }
    
      }

\[b\]\[color=green\]\[size=large\]在ParseSegment里面的run方法改动：\[/size\]  
\[/color\]\[/b\]

public int run(String[] args) throws Exception {
       
        Path segment;
    
        String usage = "Usage: ParseSegment segment [-noFilter] [-noNormalize]";
    
    //    if (args.length == 0) {
       
    //      System.err.println(usage);
    //      System.exit(-1);
    //    }
    
         args=new String[]{"D:\\20140520nutchplugin\\apache-nutch-1.8\\myseg\\20140520120541"};
        if(args.length > 1) {
       
          for(int i = 1; i < args.length; i++) {
       
            String param = args[i];
    
            if("-nofilter".equalsIgnoreCase(param)) {
       
              getConf().setBoolean("parse.filter.urls", false);
            } else if ("-nonormalize".equalsIgnoreCase(param)) {
       
              getConf().setBoolean("parse.normalize.urls", false);
            }
          }
        }
    
        segment = new Path(args[0]);
        parse(segment);
        return 0;
      }

\[color=olive\]\[size=large\]在IndexingJob的run方法里面改动：\[/size\]\[/color\]

public int run(String[] args) throws Exception {
       
        	args=new String[]{"mydir","D:\\20140520nutchplugin\\apache-nutch-1.8\\myseg\\20140520120541"};
            if (args.length < 2) {
       
                System.err
                        .println("Usage: Indexer <crawldb> [-linkdb <linkdb>] [-params k1=v1&k2=v2...] (<segment> ... | -dir <segments>) [-noCommit] [-deleteGone] [-filter] [-normalize]");
                IndexWriters writers = new IndexWriters(getConf());
                System.err.println(writers.describe());
                return -1;
            }
    
            final Path crawlDb = new Path(args[0]);
            Path linkDb = null;
    
            final List<Path> segments = new ArrayList<Path>();
            String params = null;
    
            boolean noCommit = false;
            boolean deleteGone = false;
            boolean filter = false;
            boolean normalize = false;
    
            for (int i = 1; i < args.length; i++) {
       
                if (args[i].equals("-linkdb")) {
       
                    linkDb = new Path(args[++i]);
                } else if (args[i].equals("-dir")) {
       
                    Path dir = new Path(args[++i]);
                    FileSystem fs = dir.getFileSystem(getConf());
                    FileStatus[] fstats = fs.listStatus(dir,
                            HadoopFSUtil.getPassDirectoriesFilter(fs));
                    Path[] files = HadoopFSUtil.getPaths(fstats);
                    for (Path p : files) {
       
                        segments.add(p);
                    }
                } else if (args[i].equals("-noCommit")) {
       
                    noCommit = true;
                } else if (args[i].equals("-deleteGone")) {
       
                    deleteGone = true;
                } else if (args[i].equals("-filter")) {
       
                    filter = true;
                } else if (args[i].equals("-normalize")) {
       
                    normalize = true;
                } else if (args[i].equals("-params")) {
       
                    params = args[++i];
                } else {
       
                    segments.add(new Path(args[i]));
                }
            }
    
            try {
       
                index(crawlDb, linkDb, segments, noCommit, deleteGone, params,
                        filter, normalize);
                return 0;
            } catch (final Exception e) {
       
                LOG.error("Indexer: " + StringUtils.stringifyException(e));
                return -1;
            }
        }

\[color=green\]\[size=large\]除此之外，还需要，在SolrIndexWriter的187行和SolrUtils的54行分别添加如下代码，修改solr的映射地址：\[/size\]\[/color\]

String serverURL = conf.get(SolrConstants.SERVER_URL);
            serverURL="http://localhost:8983/solr/";

// String serverURL = job.get(SolrConstants.SERVER_URL);
        String serverURL ="http://localhost:8983/solr";

\[b\]\[color=green\]\[size=large\]按上面几个步骤，每执行一个类的时候，就修改其的运行参数，因为nutch的作业具有依赖性，这一个作业的输入，往往是上一个作业的输出，手动依次运行修改上面的5个类，最终我们的索引就可以生成在solr里，截图如下：\[/size\]  
\[/color\]\[/b\]  
  
\[img\]http://dl2.iteye.com/upload/attachment/0097/3216/ea040145-4cb7-3a72-8ecc-62e27df73922.jpg\[/img\]  
  
\[b\]\[color=green\]\[size=large\]当然，我们还可以，配置分词策略，来使我们检索更加通用，准确.\[/size\]\[/color\]\[/b\]