package org.ijsolutions.pdf;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.io.StringReader;
import java.io.Writer;
import java.util.ArrayList;
import java.util.Iterator;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.pdfbox.cos.COSDocument;
import org.apache.pdfbox.io.RandomAccessFile;
import org.apache.pdfbox.pdfparser.PDFParser;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.interactive.action.PDActionGoTo;
import org.apache.pdfbox.pdmodel.interactive.documentnavigation.destination.PDNamedDestination;
import org.apache.pdfbox.pdmodel.interactive.documentnavigation.destination.PDPageDestination;
import org.apache.pdfbox.pdmodel.interactive.documentnavigation.outline.PDDocumentOutline;
import org.apache.pdfbox.pdmodel.interactive.documentnavigation.outline.PDOutlineItem;
import org.apache.pdfbox.pdmodel.interactive.documentnavigation.outline.PDOutlineNode;
import org.apache.pdfbox.text.PDFTextStripper;
import org.dom4j.Element;
import org.entermediadb.asset.Asset;
import org.entermediadb.asset.Category;
import org.entermediadb.asset.MediaArchive;
import org.entermediadb.asset.scanner.Parse;
import org.entermediadb.asset.scanner.PdfParser;
import org.entermediadb.elasticsearch.categories.ElasticCategorySearcher;
import org.ijsolutions.testbench.Question;
import org.ijsolutions.testbench.TestBench;
import org.ijsolutions.testbench.TestPlan;
import org.ijsolutions.testbench.modules.BaseTestingModule;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.openedit.Data;
import org.openedit.WebPageRequest;
import org.openedit.data.Searcher;
import org.openedit.generators.Output;
import org.openedit.hittracker.HitTracker;
import org.openedit.hittracker.SearchQuery;
import org.openedit.page.Page;
import org.openedit.page.PageRequestKeys;
import org.openedit.repository.ContentItem;
import org.openedit.repository.filesystem.FileItem;
import org.openedit.users.User;
import org.openedit.util.FileUtils;
import org.openedit.util.OutputFiller;
import org.openedit.util.PathUtilities;
import org.openedit.util.RequestUtils;
import org.openedit.util.URLUtilities;
import org.openedit.util.XmlUtil;

import com.itextpdf.text.Document;
import com.itextpdf.text.Font;
import com.itextpdf.text.pdf.PdfWriter;
import com.itextpdf.tool.xml.XMLWorker;
import com.itextpdf.tool.xml.XMLWorkerHelper;
import com.itextpdf.tool.xml.css.CssFile;
import com.itextpdf.tool.xml.css.StyleAttrCSSResolver;
import com.itextpdf.tool.xml.html.Tags;
import com.itextpdf.tool.xml.parser.XMLParser;
import com.itextpdf.tool.xml.pipeline.css.CSSResolver;
import com.itextpdf.tool.xml.pipeline.css.CssResolverPipeline;
import com.itextpdf.tool.xml.pipeline.end.PdfWriterPipeline;
import com.itextpdf.tool.xml.pipeline.html.HtmlPipeline;
import com.itextpdf.tool.xml.pipeline.html.HtmlPipelineContext;

public class PDFModule extends BaseTestingModule {

	protected RequestUtils fieldRequestUtils;
	OutputFiller filler = new OutputFiller();
	private static final Log log = LogFactory.getLog(PDFModule.class);



	public RequestUtils getRequestUtils() {
		return fieldRequestUtils;
	}

	public void setRequestUtils(RequestUtils inRequestUtils) {
		fieldRequestUtils = inRequestUtils;
	}

	public void loadPdfUtil(WebPageRequest inReq) {

		QRUtil pdf = new QRUtil();
		pdf.setRoot(getRoot());
		pdf.setPageManager(getPageManager());
		inReq.putPageValue("pdf", pdf);
	}

	public void generatePDF(WebPageRequest inReq) throws Exception {
		PdfUtil pdf = new PdfUtil();
		XmlUtil util = new XmlUtil();
		pdf.setRoot(getRoot());
		pdf.setPageManager(getPageManager());
		String language = inReq.findValue("language");
		if (language == null) {
			language = inReq.getLocale();
		}
		boolean flatten = Boolean.parseBoolean(inReq.findValue("flatten"));
		boolean lock = Boolean.parseBoolean(inReq.findValue("lock"));
		PDFInstructions instructions = new PDFInstructions();
		instructions.setFlatten(flatten);
		instructions.setLock(lock);
		// this is a path to use for the request - to call appropriate actions (xml is
		// alltypes=true so you'd miss stuff)
		String path = inReq.findValue("actionpath");
		if (path == null) {
			path = inReq.getPathUrl();
		}

		Element root = (Element) inReq.getPageValue("xmlmap");
		String inputfile = inReq.getPage().getProperty("inputpdf", language);
		Page page = getPageManager().getPage(inputfile);
		if (page == null || !page.exists()) {
			inputfile = inReq.findValue("inputpdf");
			page = getPageManager().getPage(inputfile);
		}
		if (root == null) {

			String mapfile = inReq.getPage().getProperty("mapfile", language);

			User user = inReq.getUser();
			URLUtilities urlUtil = (URLUtilities) inReq.getPageValue(PageRequestKeys.URL_UTILITIES);

			// WebPageRequest req = getRequestUtils().createVirtualPageRequest(path,
			// inReq.getUser(), urlUtil);

			WebPageRequest req = getRequestUtils().createPageRequest(getPageManager().getPage(path), inReq.getRequest(),
					inReq.getResponse(), inReq.getUser(),
					(URLUtilities) inReq.getPageValue(PageRequestKeys.URL_UTILITIES));

			req.putProtectedPageValue(PageRequestKeys.HOME, urlUtil.relativeHomePrefix());
			user = req.getUser();
			ByteArrayOutputStream scapture = new ByteArrayOutputStream();
			Writer capture = null;
			capture = new OutputStreamWriter(scapture, page.getCharacterEncoding());
			Output out = new Output(capture, scapture);
			Page map = getPageManager().getPage(mapfile);

			map.generate(req, out);
			String output = scapture.toString();
			ByteArrayInputStream in = new ByteArrayInputStream(scapture.toByteArray());

			root = util.getXml(in, page.getCharacterEncoding());

		}

		pdf.generatePDF(page.getInputStream(), inReq.getOutputStream(), root, instructions);

	}

	/**
	 * note: html to pdf does not work well see
	 * http://demo.itextsupport.com/xmlworker/ for demo cannot position table
	 * properly, nor can we justify elements to the right
	 * 
	 * @param inReq
	 * @throws Exception
	 */
	public void htmlToPdf(WebPageRequest inReq) throws Exception {
		// PdfUtil pdf = new PdfUtil();
		XmlUtil util = new XmlUtil();
		// pdf.setRoot(getRoot());
		// pdf.setPageManager(getPageManager());
		String language = inReq.findValue("language");
		if (language == null) {
			language = inReq.getLocale();
		}
		boolean flatten = Boolean.parseBoolean(inReq.findValue("flatten"));
		boolean lock = Boolean.parseBoolean(inReq.findValue("lock"));
		PDFInstructions instructions = new PDFInstructions();
		instructions.setFlatten(flatten);
		instructions.setLock(lock);
		// this is a path to use for the request - to call appropriate actions (xml is
		// alltypes=true so you'd miss stuff)
		String path = inReq.findValue("actionpath");
		if (path == null) {
			path = inReq.getPathUrl();
		}

		String inputfile = inReq.getPage().getProperty("inputfile", language);
		Page page = getPageManager().getPage(inputfile);
		String cssfile = inReq.getPage().getProperty("cssfile", language);
		Page csspage = getPageManager().getPage(cssfile);

		User user = inReq.getUser();
		URLUtilities urlUtil = (URLUtilities) inReq.getPageValue(PageRequestKeys.URL_UTILITIES);

		// WebPageRequest req = getRequestUtils().createVirtualPageRequest(path,
		// inReq.getUser(), urlUtil);
		//
		//
		// //WebPageRequest req =
		// getRequestUtils().createPageRequest(getPageManager().getPage(path),
		// inReq.getRequest(), inReq.getResponse(), inReq.getUser(),
		// (URLUtilities)inReq.getPageValue(PageRequestKeys.URL_UTILITIES));
		//
		// req.putProtectedPageValue(PageRequestKeys.HOME,
		// urlUtil.relativeHomePrefix());
		// user = req.getUser();
		ByteArrayOutputStream scapture = new ByteArrayOutputStream();
		Writer capture = null;
		capture = new OutputStreamWriter(scapture, page.getCharacterEncoding());
		Output out = new Output(capture, scapture);

		page.generate(inReq, out);
		String output = scapture.toString();
		ByteArrayInputStream in = new ByteArrayInputStream(scapture.toByteArray());

		String outputfilename = inReq.getRequestParameter("output");
		if (outputfilename == null) {
			outputfilename = inReq.findValue("output");

		}
		Page outputfile = getPageManager().getPage(outputfilename);
		Page folder = getPageManager().getPage(outputfile.getParentPath());
		File file = new File(folder.getContentItem().getAbsolutePath());
		file.mkdirs();

		Document document = new Document();
		// step 2
		PdfWriter writer = PdfWriter.getInstance(document,
				new FileOutputStream(outputfile.getContentItem().getAbsolutePath()));
		// step 3
		document.open();
		// step 4

		// step 4

		// CSS
		CSSResolver cssResolver = new StyleAttrCSSResolver();
		CssFile cssFile = XMLWorkerHelper.getCSS(new FileInputStream(csspage.getContentItem().getAbsolutePath()));
		cssResolver.addCss(cssFile);

		// HTML
		HtmlPipelineContext htmlContext = new HtmlPipelineContext(null);
		htmlContext.setTagFactory(Tags.getHtmlTagProcessorFactory());

		// Pipelines
		PdfWriterPipeline pdf = new PdfWriterPipeline(document, writer);
		HtmlPipeline html = new HtmlPipeline(htmlContext, pdf);
		CssResolverPipeline css = new CssResolverPipeline(cssResolver, html);

		// XML Worker
		XMLWorker worker = new XMLWorker(css, true);
		XMLParser p = new XMLParser(worker);
		p.parse(new StringReader(output));

		// step 5
		document.close();
		boolean redirect = Boolean.parseBoolean(inReq.findValue("redirect"));
		if (redirect) {
			inReq.redirect(outputfilename);
		}
		inReq.putPageValue("output", outputfilename);

	}

	public void testPDFExtracrtion(WebPageRequest inReq) {
		MediaArchive archive = getMediaArchive(inReq);
		Asset asset = getAsset(inReq);
		ContentItem item = archive.getOriginalContent(asset);
		extractData(archive, item, asset);

	}

	public boolean extractData(MediaArchive inArchive, ContentItem inFile, Asset inAsset) {

		String type = PathUtilities.extractPageType(inFile.getPath());
		if (type == null || "data".equals(type.toLowerCase())) {
			type = inAsset.get("fileformat");
		}

		if (type != null) {
			type = type.toLowerCase();
			if (inAsset.get("fileformat") == null) {
				inAsset.setProperty("fileformat", type);
			}
			if (type.equals("pdf")) {
				// log.info("Extracting Metadata from PDF");
				PdfParser parser = new PdfParser();

//				ByteArrayOutputStream out = new ByteArrayOutputStream();
				InputStream in = null;
				try {
					long maxsize = 100000000;
					String sizeval = inArchive.getCatalogSettingValue("maxpdfsize");
					if (sizeval != null) {
						maxsize = Long.valueOf(sizeval);
					}
					if (inFile.getLength() > maxsize) {
						log.info("PDF was too large to extract metadata. Consider increasing max size: " + sizeval);
						return false;
					}

					in = inFile.getInputStream();
//					try
//					{
//						new OutputFiller().fill(in, out);
//					}
//					finally
//					{
//						FileUtils.safeClose(in);
//					}
//					byte[] bytes = out.toByteArray();

					Parse results = parser.parse(in); // Do we deal with encoding?
					// We need to limit this size
					String fulltext = results.getText();
					if (fulltext != null && fulltext.length() > 0) {

						ContentItem item = getPageManager().getRepository().getStub("/WEB-INF/data/"
								+ inArchive.getCatalogId() + "/assets/" + inAsset.getSourcePath() + "/fulltext.txt");
						if (item instanceof FileItem) {
							((FileItem) item).getFile().getParentFile().mkdirs();
						}
						PrintWriter output = new PrintWriter(item.getOutputStream());
						filler.fill(new StringReader(fulltext), output);
						filler.close(output);
						inAsset.setProperty("hasfulltext", "true");
					}

					if (inAsset.getInt("width") == 0) {
						String val = results.get("width");
						inAsset.setProperty("width", val);
					}
					if (inAsset.getInt("height") == 0) {
						String val = results.get("height");
						inAsset.setProperty("height", val);
					}
					String pages = String.valueOf(results.getPages());
					inAsset.setProperty("pages", pages);
					log.info("PDF pages: " + pages);
					if (inAsset.get("assettitle") == null) {
						String title = results.getTitle();
						if (title != null && title.length() < 300) {
							inAsset.setProperty("assettitle", title);
						}
					}

				} catch (Exception ex) {
					log.info("cant process", ex);
					return false;
				} finally {
					FileUtils.safeClose(in);
				}

				return true;
			}
		}
		return false;
	}



	public void loadPDF(WebPageRequest inReq) {
		TestBench bench = getTestBench(inReq);

		String assetid = inReq.findValue("assetid");
		String pagenumber = inReq.findValue("pagenumber");
		String searchtext = inReq.findValue("searchtext");
		
		
		
		TestPlan testplan = null;
		String planid = inReq.findValue("planid");
		if (planid != null) {
			testplan = bench.getTestPlan(planid);
			if (testplan.get("document") != null) {
				assetid = testplan.get("document");
			}
			if (testplan.get("startingpage") != null) {
				pagenumber = testplan.get("startingpage");
			}
		}
		Question q = null;
		String questionid = inReq.findValue("questionid");
		if (questionid != null) {
			q = bench.getQuestion(questionid);
			if (q.get("pdftext") != null) {
				Data reference = bench.getData("pdftext", q.get("pdftext"));
				assetid = reference.get("assetid");
				if (pagenumber != null) {
					pagenumber = reference.get("pagenumber");
				}
			}
			if (assetid == null) {
				assetid = q.get("media");
			}

		}

		MediaArchive archive = getMediaArchive(inReq);
		Asset asset = archive.getAsset(assetid);
		inReq.putPageValue("pdf", asset);
		Searcher pdftextSearcher = archive.getSearcher("pdftext");
		if (asset != null) {
			HitTracker allhits = pdftextSearcher.query().exact("assetid", asset.getId()).named("pdfpages")
					.sort("pagenumber").search(inReq, 15);
			if(allhits.size() == 0) {
				try {
					//TODO: Run in seperate thread
					extractPDFPageText(inReq);
					extractBookmarks(inReq);
					
					allhits = pdftextSearcher.query().exact("assetid", asset.getId()).named("pdfpages")
							.sort("pagenumber").search(inReq, 15);
					
				} catch (Exception e) {
					// TODO Auto-generated catch block
					e.printStackTrace();
				}
				
			}
			if (pagenumber != null) {
				Integer page = Integer.parseInt(pagenumber);
				int location = allhits.getPageForIndexLocation(page);
				allhits.setPage(location);
				inReq.putPageValue("pagenumber", pagenumber);
				Data bookmark = findClosestBookmark(archive, asset, page);
				inReq.putPageValue("bestbookmark", bookmark);
				log.info("Best bookmark was: " + bookmark);

			}

			String increment = inReq.findValue("increment");
			if (Boolean.parseBoolean(increment)) {
				int page = allhits.nextPage();
				allhits.setPage(page);

			}
		}
		bench.fireLearningEvent(inReq.getUser(), "viewed", q, "question");

	}

	public void searchPDF(WebPageRequest inReq) {
		TestBench bench = getTestBench(inReq);
		MediaArchive archive = getMediaArchive(inReq);

		TestPlan testplan = null;
		Searcher pdftextSearcher = archive.getSearcher("pdftext");

		String planid = inReq.findValue("planid");
		if (planid != null) {
			String assetid = inReq.findValue("assetid");

			testplan = bench.getTestPlan(planid);
			if (testplan.get("document") != null) {
				assetid = testplan.get("document");
			}
			Asset asset = archive.getAsset(assetid);
			inReq.putPageValue("pdf", asset);
			String searchterm = testplan.get("documentkeywords");

			inReq.putPageValue("searchterm", searchterm);

			HitTracker searchresults = pdftextSearcher.query().exact("assetid", asset.getId()).named("hits")
					.sort("pagenumber").contains("pagetext", searchterm).search(inReq, 15);

			if (searchresults.size() == 1) {
				Data hit = (Data) searchresults.first();
				inReq.putPageValue("pagenumber", hit.get("pagenumber"));
				inReq.setRequestParameter("pagenumber", hit.get("pagenumber"));
			}

		} else {
			String assetid = inReq.findValue("assetid");
			if (assetid != null) {
				Asset asset = archive.getAsset(assetid);
				String field[] = inReq.getRequestParameters("field");
				if (field != null) {
					SearchQuery q = pdftextSearcher.addStandardSearchTerms(inReq);
					if (asset != null) {
						q.addExact("assetid", asset.getId());
					} else {
					
					}
					q.addSortBy("assetid");
					q.addSortBy("pagenumber");

					HitTracker searchresults = pdftextSearcher.cachedSearch(inReq, q);
					if (searchresults.size() == 1) {

						Data hit = (Data) searchresults.first();
						inReq.putPageValue("pagenumber", hit.get("pagenumber"));
						inReq.setRequestParameter("pagenumber", hit.get("pagenumber"));
					}
				}
			}
		}

	}

	public Data findClosestBookmark(MediaArchive inArchive, Asset inAsset, Integer inPage) {

		Data bestbookmark = (Data) inArchive.getCacheManager().get("bookmarks", inAsset.getId() + inPage);
		if (bestbookmark != null) {
			return bestbookmark;
		}

		ElasticCategorySearcher searcher = (ElasticCategorySearcher) inArchive.getSearcher("pdfbookmark");
		HitTracker allhits = searcher.query().match("id", "*").exact("assetid", inAsset.getId()).sort("pagenumber")
				.search();
		allhits.enableBulkOperations();// could be more than 10000 pages

		Data closest = (Data) allhits.first();
		if (closest == null) {
			return null;
		}
		for (Iterator iterator = allhits.iterator(); iterator.hasNext();) {
			Data hit = (Data) iterator.next();
			Integer page = (Integer) hit.getValue("pagenumber");

			if (page != null) {
				Integer current = (Integer) closest.getValue("pagenumber");
				if (current < inPage) {
					Integer next = (Integer) hit.getValue("pagenumber");
					if (next < inPage) {
						closest = hit;
					} else {
						break;
					}
					log.info("Set closest to " + closest.get("pagenumber"));
				} else {
					break;
				}
			}
		}

		inArchive.getCacheManager().put("bookmarks", inAsset.getId() + inPage, closest);

		return closest;

	}

	public void loadBookmarkJson(WebPageRequest inReq) {
		String assetid = inReq.findValue("assetid");
		MediaArchive archive = getMediaArchive(inReq);
		Asset asset = archive.getAsset(assetid);
		ElasticCategorySearcher searcher = (ElasticCategorySearcher) archive.getSearcher("pdfbookmark");
		searcher.setSort("pagenumber");
		String catid = inReq.getRequestParameter("id");
		Category bookroot = null;
		if (catid != null && !"#".equals(catid)) {
			bookroot = (Category) searcher.query().match("id", catid).searchOne();
		} else {
			String bookmarkroot = asset.get("pdfbookmarkroot");

			if (bookmarkroot != null) {
				bookroot = searcher.getCategory(bookmarkroot);
			}
		}
		String pagenumber = inReq.findValue("pagenumber");

		JSONObject root = new JSONObject();
		root.put("id", bookroot.getId());
		root.put("text", bookroot.getName());
		JSONObject attrib = new JSONObject();
		attrib.put("pagenumber", bookroot.get("pagenumber"));
		root.put("li_attr", attrib);

		Data bookmark = null;

		if (pagenumber != null) {
			Integer page = Integer.parseInt(pagenumber);
			bookmark = findClosestBookmark(archive, asset, page);
			inReq.putPageValue("bestbookmark", bookmark);
			log.info("Best bookmark was: " + bookmark);
			Category cat = searcher.getCategory(bookmark.getId());
			JSONObject state = new JSONObject();
			if (bookroot.isAncestorOf(cat)) {
				state.put("opened", true);
			} else {
				state.put("opened", false);
			}
			if (bookroot.getId().equals(cat.getId())) {
				state.put("opened", true);
				state.put("selected", true);
			}

			root.put("state", state);

		}

		if (bookroot.hasChildren()) {
			root.put("children", true);
			JSONArray children = new JSONArray();
			root.put("children", children);

			for (Iterator iterator = bookroot.getChildren().iterator(); iterator.hasNext();) {
				Category child = (Category) iterator.next();
				JSONObject data = new JSONObject();
				data.put("id", child.getId());
				data.put("text", child.getName());
				if (child.hasChildren()) {
					data.put("children", true);
				}

				if (pagenumber != null) {
					Integer page = Integer.parseInt(pagenumber);
					Category cat = searcher.getCategory(bookmark.getId());
					JSONObject state = new JSONObject();
					if (child.isAncestorOf(cat)) {
						state.put("opened", true);
					} else {
						state.put("opened", false);
					}
					if (child.getId().equals(cat.getId())) {
						state.put("opened", true);
						state.put("selected", true);
					}

					data.put("state", state);

				}

				JSONObject attribs = new JSONObject();
				attribs.put("pagenumber", child.get("pagenumber"));
				data.put("li_attr", attribs);

				children.add(data);

			}
		}
		inReq.putPageValue("json", root.toJSONString());
//		[
//			{
//				"id": 1,
//				"text": "Root node",
//				"children": [
//					{
//						"id": 2,
//						"text": "Child node 1",
//						"children": true
//					},
//					{
//						"id": 3,
//						"text": "Child node 2"
//					}
//				]
//			}
//		]		

		if (bookroot != null) {

		}

	}

	public void extractPDFPageText(WebPageRequest inReq) throws Exception {
		String assetid = inReq.findValue("assetid");
		MediaArchive archive = getMediaArchive(inReq);
		Asset asset = archive.getAsset(assetid);
		ContentItem item = archive.getOriginalContent(asset);
		Searcher pdftextSearcher = archive.getSearcher("pdftext");

		int pages = asset.getInt("pages");
		ArrayList datatosave = new ArrayList();

		PDFTextStripper pdfStripper = null;
		PDDocument pdDoc = null;
		PDFParser parser = new PDFParser(new RandomAccessFile(new File(item.getAbsolutePath()), "r"));
		parser.parse();
		pdfStripper = new PDFTextStripper();
		COSDocument cosDoc = parser.getDocument();
		pdDoc = new PDDocument(cosDoc);
		pages = pdDoc.getNumberOfPages();
		for (int i = 0; i < pages; i++) {
			if (asset != null) {

				PdfUtil pdf = new PdfUtil();
				pdf.setRoot(getRoot());
				pdf.setPageManager(getPageManager());
				Data current = pdftextSearcher.query().exact("pagenumber", String.valueOf(i)).exact("assetid", assetid)
						.searchOne();
				if (current == null) {
					current = pdftextSearcher.createNewData();
					current.setValue("assetid", assetid);
					current.setValue("pagenumber", i);
					current.setValue("name", asset.getName() + " : " + i);

					pdfStripper.setStartPage(i);
					pdfStripper.setEndPage(i);
					String parsedText = pdfStripper.getText(pdDoc);
					// log.info(parsedText);
					log.info("Parsed page" + i + "of PDF " + asset.getName());
					current.setValue("pagetext", parsedText);
					datatosave.add(current);
				}

			}
			if (datatosave.size() > 1000) {
				pdftextSearcher.saveAllData(datatosave, inReq.getUser());
				datatosave.clear();
			}
		}

		pdftextSearcher.saveAllData(datatosave, inReq.getUser());
	}

	public void autoLinkQuestions(WebPageRequest inReq) throws Exception {
		String assetid = inReq.findValue("assetid");
		MediaArchive archive = getMediaArchive(inReq);
		TestBench tb = getTestBench(inReq);

		Searcher pdftextSearcher = archive.getSearcher("pdftext");
		Searcher questionsearcher = archive.getSearcher("question");
		int count = 0;
		HitTracker tolink = questionsearcher.query().exists("reference").search();
		for (Iterator iterator = tolink.iterator(); iterator.hasNext();) {
			Data question = (Data) iterator.next();
			String reference = question.get("reference");
			Data pagewithtext = pdftextSearcher.query().startsWith("description", reference).searchOne();
			if (pagewithtext != null) {
				Question real = tb.getQuestion(question.getId());
				real.setValue("pdftext", pagewithtext.getId());
				questionsearcher.saveData(real);
				count++;
			}
		}

		ArrayList datatosave = new ArrayList();
		inReq.putPageValue("count", count);

		pdftextSearcher.saveAllData(datatosave, inReq.getUser());
	}

	public void extractBookmarks(WebPageRequest inReq) throws Exception {
		String assetid = inReq.findValue("assetid");
		MediaArchive archive = getMediaArchive(inReq);
		TestBench tb = getTestBench(inReq);
		ArrayList bookmarks = new ArrayList();

		Asset asset = archive.getAsset(assetid);
		ContentItem item = archive.getOriginalContent(asset);

		PDDocument pdDoc = null;
		PDFParser parser = new PDFParser(new RandomAccessFile(new File(item.getAbsolutePath()), "r"));
		parser.parse();

		COSDocument cosDoc = parser.getDocument();
		pdDoc = new PDDocument(cosDoc);
		PDDocumentOutline bookmark = pdDoc.getDocumentCatalog().getDocumentOutline();
		ElasticCategorySearcher searcher = (ElasticCategorySearcher) archive.getSearcher("pdfbookmark");

		String bookmarkroot = asset.get("pdfbookmarkroot");
		Category bookroot = null;
		if (bookmarkroot != null) {
			bookroot = searcher.getCategory(bookmarkroot);
			searcher.deleteCategoryTree(bookroot);
		}

		bookroot = searcher.createCategoryPath("documents/" + assetid);
		bookroot.setName(asset.getName());
		asset.setValue("pdfbookmarkroot", bookroot.getId());
		archive.saveAsset(asset);

		searcher.saveData(bookroot);
		bookroot.setValue("assetid", assetid);
		bookroot.setValue("isroot", true);
		if (bookmark != null) {
			printBookmark(asset, searcher, bookroot, pdDoc, bookmark, "");
		}

	}

	public void printBookmark(Asset inAsset, ElasticCategorySearcher searcher, Category root, PDDocument document,
			PDOutlineNode bookmark, String indentation) throws Exception {
		PDOutlineItem current = bookmark.getFirstChild();
		while (current != null) {
			Category nextchild = (Category) searcher.createNewData();
			nextchild.setParentCategory(root);
			nextchild.setValue("assetid", inAsset.getId());

			root.addChild(nextchild);
			int destinationpage = -1;
			if (current.getDestination() instanceof PDPageDestination) {
				PDPageDestination pd = (PDPageDestination) current.getDestination();
				int page = pd.retrievePageNumber() + 1;
				System.out.println(indentation + "Destination page: " + page);
				destinationpage = page;
			} else if (current.getDestination() instanceof PDNamedDestination) {
				PDPageDestination pd = document.getDocumentCatalog()
						.findNamedDestinationPage((PDNamedDestination) current.getDestination());
				if (pd != null) {
					int page = pd.retrievePageNumber() + 1;
					System.out.println(indentation + "Destination page: " + page);
					destinationpage = page;
				}
			} else if (current.getDestination() != null) {
				System.out.println(
						indentation + "Destination class: " + current.getDestination().getClass().getSimpleName());
			}

			if (current.getAction() instanceof PDActionGoTo) {
				PDActionGoTo gta = (PDActionGoTo) current.getAction();
				if (gta.getDestination() instanceof PDPageDestination) {
					PDPageDestination pd = (PDPageDestination) gta.getDestination();
					int page = pd.retrievePageNumber() + 1;
					System.out.println(indentation + "Destination page: " + page);
					destinationpage = page;
				} else if (gta.getDestination() instanceof PDNamedDestination) {
					PDPageDestination pd = document.getDocumentCatalog()
							.findNamedDestinationPage((PDNamedDestination) gta.getDestination());
					if (pd != null) {
						int page = pd.retrievePageNumber() + 1;
						System.out.println(indentation + "Destination page: " + page);
						destinationpage = page;
					}
				} else {
					System.out.println(
							indentation + "Destination class: " + gta.getDestination().getClass().getSimpleName());
				}
			} else if (current.getAction() != null) {
				System.out.println(indentation + "Action class: " + current.getAction().getClass().getSimpleName());
			}
			if (destinationpage != -1) {
				nextchild.setValue("pagenumber", destinationpage);
				nextchild.setName(current.getTitle());
				searcher.saveCategory(nextchild);
			}

			System.out.println(indentation + current.getTitle());
			printBookmark(inAsset, searcher, nextchild, document, current, indentation + "    ");
			current = current.getNextSibling();
		}
	}

}