package org.openedit.sitesearch.parse; import java.io.IOException; import java.io.StringWriter; import java.io.Writer; import java.util.ArrayList; import java.util.Iterator; import java.util.List; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.entermediadb.links.Link; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.safety.Whitelist; import org.jsoup.select.Elements; import org.openedit.OpenEditException; import org.openedit.sitesearch.Content; import org.openedit.sitesearch.Parse; import org.openedit.sitesearch.Parser; import org.openedit.util.PathUtilities; public class HtmlParser implements Parser { private static final Log log = LogFactory.getLog(HtmlParser.class); public Parse getParse(Content inUrl) throws OpenEditException { log.info("parse " + inUrl.getUrl()); Parse results = new Parse(); Document doc; try { doc = Jsoup.connect(inUrl.getUrl().getPath()).userAgent("Mozilla").get(); } catch (IOException e) { throw new OpenEditException(e); } Writer l = new StringWriter(); // source.setLogWriter(l); // send log messages to stderr // // source.fullSequentialParse(); //System.out.println("Document title:"); String title=doc.title(); results.setTitle(title); //System.out.println(title==null ? "(none)" : title); //System.out.println("\nDocument keywords:"); Elements select = doc.select("meta[name=keywords]"); Element first = select.first(); String keywords = null; if(first != null){ keywords=first.attr("content"); } //System.out.println(keywords==null ? "(none)" : keywords); if( keywords != null) { keywords = keywords.replace(","," "); } //TODO: Each time someone links to this page maybe we should add to a list of links if( inUrl.getUrl().getText() != null) { if( keywords == null) { keywords = inUrl.getUrl().getText(); } else { keywords = inUrl.getUrl().getText() + " " + keywords; } } results.put( "keywords", keywords); //System.out.println("\nDocument description:"); String description=getMetaValue(doc,"description"); //System.out.println(description==null ? "(none)" : description); results.put( "summary", description); //System.out.println("\nLinks to other documents:"); Elements linkElements = doc.select("a[href]"); List links = new ArrayList(); String hostName = inUrl.getUrl().getPath(); hostName = hostName.substring(0, hostName.indexOf("/",8)); for (Iterator i=linkElements.iterator(); i.hasNext();) { Element linkElement=(Element)i.next(); String href=linkElement.attr("href"); if (href==null) continue; String follow = linkElement.attr("rel"); if( follow != null && "nofollow".equalsIgnoreCase(follow)) { continue; } //System.out.println(href+" ("+label+")"); if( href.length() < 2 || href.startsWith("#") || href.indexOf("?") > -1) { continue; } int pound = href.indexOf("#"); if ( pound > -1) { href = href.substring(0,pound); } if( href.length() < 2) { continue; } // A element can contain other tags so need to extract the text from it: Link thelink = new Link(); String label=linkElement.text(); thelink.setText(label); if( href.startsWith("http") ) { thelink.setPath(href); } else if( href.startsWith("/")) { thelink.setPath(hostName + href); } else { String cleanhref = PathUtilities.resolveRelativePath(href,inUrl.toString()); thelink.setPath(hostName + cleanhref); } links.add(thelink); } results.put( "links", links); List images = new ArrayList(); Elements img = doc.select("img[src~=(?i)\\.(png|jpe?g|gif)]"); for (Element image : img) { Link link = new Link(); String src = image.attr("src"); if( src.startsWith("http") ) { link.setPath(src); } else if( src.startsWith("/")) { link.setPath(hostName + src); } else { String cleanhref = PathUtilities.resolveRelativePath(src,inUrl.toString()); link.setPath(hostName + cleanhref); } String alt = image.attr("alt"); link.setText(alt); images.add(link); } results.put( "images", images); //System.out.println("\nAll text from BODY (exluding content inside SCRIPT and STYLE elements):"); Element bodyElement= doc.body(); String text = Jsoup.clean(bodyElement.text(), Whitelist.basicWithImages()); results.setText( text); return results; } private static String getMetaValue(Document doc, String key) { Elements select = doc.select("meta[name="+key+"]"); if(select.first() != null){ return select.first().attr("content"); } return null; } }