package org.openedit.sitesearch; import java.io.File; import java.net.HttpURLConnection; import java.net.URL; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.dom4j.Element; import org.entermediadb.asset.MediaArchive; import org.entermediadb.asset.SourcePathCreator; import org.entermediadb.elasticsearch.searchers.BaseElasticSearcher; import org.entermediadb.links.Link; import org.entermediadb.modules.update.Downloader; import org.openedit.ModuleManager; import org.openedit.OpenEditException; import org.openedit.page.Page; import org.openedit.page.manage.PageManager; import org.openedit.util.PathUtilities; import org.openedit.util.XmlUtil; public class SiteSearcher extends BaseElasticSearcher { private static final Log log = LogFactory.getLog(SiteSearcher.class); protected ModuleManager fieldModuleManager; protected PageManager fieldPageManager; protected MediaArchive fieldMediaArchive; protected SourcePathCreator fieldSourcePathCreator; public SourcePathCreator getSourcePathCreator() { return fieldSourcePathCreator; } public void setSourcePathCreator(SourcePathCreator inSourcePathCreator) { fieldSourcePathCreator = inSourcePathCreator; } public PageManager getPageManager() { return fieldPageManager; } public void setPageManager(PageManager fieldPageManager) { this.fieldPageManager = fieldPageManager; } protected Map fieldParsers; public void reIndexAll() { //Loop over site and search for all content I can find //TODO: Add PDF Page configpage = getPageManager().getPage("/" + getCatalogId() + "/search.xml"); File config = new File( configpage.getContentItem().getAbsolutePath()); if( !config.exists() ) { throw new OpenEditException("No urls specified in " + config.getAbsolutePath() ); } try { Status status = new Status(); Element root = new XmlUtil().getXml(config,"UTF-8"); for (Iterator iterator = root.elementIterator("allow"); iterator.hasNext();) { Element site = (Element) iterator.next(); status.getAllowedSites().add(site.getTextTrim()); } String start = root.elementText("starturl"); Link startlink = new Link(); startlink.setPath(start); index(startlink,status); log.info("Indexed " + status.getVisitedHrefs().size() + " links"); } catch( Exception ex) { throw new OpenEditException(ex); } } protected void index(Link inLink, Status inStatus) throws Exception { // String inUrl = inLink.getPath(); // // inStatus.addVisitedUrl(inUrl); // Content content = new Content(); // content.setUrl(inLink); // Parse results = getParser(inUrl).getParse(content); // if( results == null) // { // log.error("No content on " + inUrl); // return; // } // Document doc = new Document(); // doc.add(new Field("href", inUrl, Field.Store.YES, Field.Index.ANALYZED)); // // String title = results.getTitle(); // if( title != null) // { // doc.add(new Field("title", title, Field.Store.YES, Field.Index.ANALYZED)); // } //// if( keywords != null) //// { //// doc.add(new Field("keywords", keywords, Field.Store.YES, Field.Index.TOKENIZED)); //// } // String summary = (String)results.get("summary"); // String body = results.getText(); // if( summary != null) // { // doc.add(new Field("summary", summary, Field.Store.YES, Field.Index.ANALYZED)); // } // StringBuffer all = new StringBuffer();//= title + " " + keywords + " " + summary + " " + body; // if( title != null) // { // all.append(title); // } // if( summary != null) // { // all.append(" "); // all.append(summary); // } // String keywords = (String)results.get("keywords"); // if( keywords != null) // { // all.append(" "); // all.append(keywords); // } // if ( body != null) // { // all.append(" "); // all.append(body); // } // // if( all.length() > 0) // { // doc.add(new Field("description", all.toString(), Field.Store.YES, Field.Index.ANALYZED)); // } // inWriter.addDocument(doc); // //Now look for links not yet visited // List links = (List)results.getList("links"); // int count = 0; // for (Iterator iterator = links.iterator(); iterator.hasNext();) // { // Link href = (Link) iterator.next(); // if( inStatus.followHref(href.getPath()) ) // { // try // { // count++; // index(href,inStatus, inWriter); // } // catch ( Exception ex) // { // log.error("Could not parse " + href + " " + ex); // } // } // } // if( count > 5) // { // log.info("sleep"); // Thread.sleep(500); // } } public MediaArchive getMediaArchive() { if (fieldMediaArchive == null) { fieldMediaArchive = (MediaArchive) getModuleManager().getBean(getCatalogId(), "mediaArchive"); } return fieldMediaArchive; } public void crawlMedia(Link inLink, Status inStatus) throws Exception { String inUrl = inLink.getPath(); inStatus.addVisitedUrl(inUrl); Content content = new Content(); content.setUrl(inLink); Parse results = getParser(inUrl).getParse(content); if( results == null) { log.error("No content on " + inUrl); return; } int imagesdownloaded; List images = (List)results.getList("images"); for (Iterator iterator = images.iterator(); iterator.hasNext();) { Link imagelink = (Link) iterator.next(); if( inStatus.followHref(imagelink.getPath())||true ){ inStatus.addVisitedUrl(inUrl); URL url = new URL(imagelink.getPath()); HttpURLConnection connection = (HttpURLConnection) url.openConnection(); int length = connection.getContentLength(); // i get negetive length if(length > 100000){ createAsset( inStatus,imagelink); } } } //Now look for links not yet visited List links = (List)results.getList("links"); int count = 0; for (Iterator iterator = links.iterator(); iterator.hasNext();) { Link href = (Link) iterator.next(); if( inStatus.followHref(href.getPath()) ) { try { count++; crawlMedia(href,inStatus); } catch ( Exception ex) { log.error("Could not parse " + href + " " + ex); } } } if( count > 5) { log.info("sleep"); Thread.sleep(500); } } protected void createAsset(Status inStatus, Link imagelink) { String sourcepath = getMediaArchive().getAssetSearcher().nextAssetNumber(); Downloader downloader = new Downloader(); String path = "/WEB-INF/data/" + getMediaArchive().getCatalogId() + "/originals/" + sourcepath; File attachments = new File(getMediaArchive().getPageManager().getPage(path) .getContentItem().getAbsolutePath()); String filename = PathUtilities.extractFileName(imagelink.getPath()); filename = filename.replaceAll("\\?.*", ""); log.info("Downloading " + imagelink.getPath() + " ->" + path + "/" + filename); File target = new File(attachments, filename); if (target.exists() || target.length() == 0) { downloader.download(imagelink.getPath(), target); } } protected Parser getParser(String inPath) { String type = null;// if( inPath.indexOf("." ) > -1 ) { type = PathUtilities.extractPageType(inPath); } else { type = "html"; } if( type != null) { type = type.toLowerCase(); if( type.equals("htm")) { type = "html"; } type = type + "Parser"; Parser parser = (Parser)getParsers().get(type); if( parser == null) { if( parser == null) { log.error("No parser for " + inPath + " using html"); parser = (Parser)getModuleManager().getBean("htmlParser"); } getParsers().put(type, parser); } return parser; } log.error("No parser for " + inPath); return null; } protected Map getParsers() { if( fieldParsers == null) { fieldParsers = new HashMap(); } return fieldParsers; } public ModuleManager getModuleManager() { return fieldModuleManager; } public void setModuleManager(ModuleManager inModuleManager) { fieldModuleManager = inModuleManager; } }