Example: PoliteSpider

• Robots processing instruction:

  <?robots index="yes | no"
           follow="yes | no" ?>

package nu.xom.samples;

import java.net.*;
import java.util.*;
import nu.xom.*;

public class PoliteSpider {

    private Set spidered   = new HashSet();
    private Builder parser = new Builder();
    private List queue     = new LinkedList();
    
    public static final String XLINK_NS 
     = "http://www.w3.org/1999/xlink";
    public static final String XML_NS 
     = "http://www.w3.org/XML/1998/namespace";
    
    public void search(URL url) {
        
        try {
            String systemID = url.toExternalForm();
            Document doc = parser.build(systemID);
            
            boolean follow = true;
            boolean index = true;
            for (int i = 0; i < doc.getChildCount(); i++) {
                Node child = doc.getChild(i); 
                if (child instanceof Element) break;  
                if (child instanceof ProcessingInstruction){
                    ProcessingInstruction instruction 
                      = (ProcessingInstruction) child;
                    if (instruction.getTarget().equals("robots")) {
                        Element data 
                          = PseudoAttributes.getAttributes(instruction); 
                        Attribute indexAtt = data.getAttribute("index"); 
                        if (indexAtt != null) {
                            String value = indexAtt.getValue().trim();
                            if (value.equals("no")) index = false;
                        }
                        Attribute followAtt = data.getAttribute("follow"); 
                        if (followAtt != null) {
                            String value = followAtt.getValue().trim();
                            if (value.equals("no")) follow = false;
                        }
                    }   
                }  
            }
            
            if (index) System.out.println(url);
            if (follow) search(doc.getRootElement(), url);
        }
        catch (Exception ex) {
            // just skip this document
        }
        
        if (queue.isEmpty()) return;
        
        URL discovered = (URL) queue.remove(0);
        spidered.add(discovered);
        search(discovered);      
        
    }

    private void search(Element element, URL base) {

        Attribute href = element.getAttribute("href", XLINK_NS); 
        Attribute xmlbase = element.getAttribute("base", XML_NS);
        try {
            if (xmlbase != null) base = new URL(base, xmlbase.getValue());
        }
        catch (MalformedURLException ex) {
            //Java can't handle the kind of URLs used inside this element
            return;
        }
        if (href != null) {
            String uri = href.getValue();
            // absolutize URL
            try {
                URL discovered = new URL(base, uri);
                // strip ref field if any
                discovered = new URL(
                  discovered.getProtocol(),
                  discovered.getHost(),
                  discovered.getFile()
                );
                
                if (!spidered.contains(discovered) 
                  && !queue.contains(discovered)) {
                    queue.add(discovered);   
                }
            }
            catch (MalformedURLException ex) {
                // skip this one   
            }
        }
        Elements children = element.getChildElements();
        for (int i = 0; i < children.size(); i++) {
            search(children.get(i), base);
        }
        
    }

    public static void main(String[] args) {
      
        XLinkSpider spider = new XLinkSpider();
        for (int i = 0; i < args.length; i++) { 
            try { 
                spider.search(new URL(args[i]));
            }
            catch (MalformedURLException ex) {
                System.err.println(ex);   
            }
        }
      
    }   // end main()

}