网络爬虫(源代码参考)

秒速五厘米 2022-09-19 14:23 212阅读 0赞

**网络爬虫（源代码参考）**

**作者：Ackarlix**

package com.heaton.bot;

import com.heaton.bot.\*;

import java.net.\*;

/\*\*

\* The SpiderWorker class performs the actual work of

\* spidering pages. It is implemented as a thread

\* that is created by the spider class.

\* @author Jeff Heaton

\* @version 1.2

\*/

public class SpiderWorker extends Thread \{

/\*\*

\* The URL that this spider worker

\* should be downloading.

\*/

protected String target;

/\*\*

\* The owner of this spider worker class,

\* should always be a Spider object.

\* This is the class that this spider

\* worker will send its data to.

\*/

protected Spider owner;

/\*\*

\* Indicates if the spider is busy or not.

\* true = busy

\* false = idle

\*/

protected boolean busy;

/\*\*

\* A descendant of the HTTP object that

\* this class should be using for HTTP

\* communication. This is usually the

\* HTTPSocket class.

\*/

protected HTTP http;

/\*\*

\* Constructs a spider worker object.

\* @param owner The owner of this object, usually

\* a Spider object.

\* @param http

\*/

public SpiderWorker(Spider owner,HTTP http)

this.http = http;

this.owner = owner;

/\*\*

\* Returns true of false to indicate if

\* the spider is busy or idle.

\* @return true = busy

\* false = idle

\*/

public boolean isBusy()

return this.busy;

/\*\*

\* The run method causes this thread to go idle

\* and wait for a workload. Once a workload is

\* received, the processWorkload method is called

\* to handle the workload.

\*/

public void run()

for ( ;; ) \{

target = this.owner.getWorkload();

if ( target==null )

return;

owner.getSpiderDone().workerBegin();

processWorkload();

owner.getSpiderDone().workerEnd();

/\*\*

\* The run method actually performs the

\* the workload assigned to this object.

\*/

public void processWorkload()

try \{

busy = true;

Log.log(Log.LOG\_LEVEL\_NORMAL,"Spidering " + target );

http.send(target,null);

Attribute typeAttribute = http.getServerHeaders().get("Content-Type");

// if no content-type at all, its PROBABLY not HTML

if ( typeAttribute==null )

return;

// now check to see if is HTML, ONLY PARSE text type files(namely HTML)

owner.processPage(http);

if ( !typeAttribute.getValue().startsWith("text/") )

return;

HTMLParser parse = new HTMLParser();

parse.source = new StringBuffer(http.getBody());

// find all the links

while ( !parse.eof() ) \{

char ch = parse.get();

if ( ch==0 ) \{

HTMLTag tag = parse.getTag();

Attribute link = tag.get("HREF");

if ( link==null )

link = tag.get("SRC");

if ( link==null )

continue;

URL target=null;

try \{

target = new URL(new URL(this.target),link.getValue());

\} catch ( MalformedURLException e ) \{

Log.log(Log.LOG\_LEVEL\_TRACE,

"Spider found other link: " + link );

owner.foundOtherLink(link.getValue());

continue;

if ( owner.getRemoveQuery() )

target = URLUtility.stripQuery(target);

target = URLUtility.stripAnhcor(target);

if ( target.getHost().equalsIgnoreCase(

new URL(this.target).getHost()) ) \{

Log.log(Log.LOG\_LEVEL\_NORMAL,

"Spider found internal link: " + target.toString() );

owner.foundInternalLink(target.toString());

\} else \{

Log.log(Log.LOG\_LEVEL\_NORMAL,

"Spider found external link: " + target.toString() );

owner.foundExternalLink(target.toString());

owner.completePage(http,false);

\} catch ( java.io.IOException e ) \{

Log.log(Log.LOG\_LEVEL\_ERROR,

"Error loading file("+ target +"): " + e );

owner.completePage(http,true);

\} catch ( Exception e ) \{

Log.logException(

"Exception while processing file("+ target +"): ", e );

owner.completePage(http,true);

\} finally \{

busy = false;

/\*\*

\* Returns the HTTP descendant that this

\* object should use for all HTTP communication.

\* @return An HTTP descendant object.

\*/

public HTTP getHTTP()

return http;

网络爬虫(源代码参考)

发表评论取消回复

还没有评论，来说两句吧...

相关阅读

相关【网络】计算机网络参考模型

相关 java爬虫(本地爬虫和网络爬虫)

相关网络爬虫(源代码)

相关网络爬虫(源代码参考)

相关爬虫参考资料

相关网络爬虫去重参考

相关网络爬虫

相关网络爬虫

相关 SpringBoot多数据源源代码参考

相关神经网络 OCR 参考