text:
package docet.maven;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileFilter;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.StandardOpenOption;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.EnumMap;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.function.BiConsumer;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;

import org.apache.commons.io.FileUtils;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.fr.FrenchAnalyzer;
import org.apache.lucene.analysis.it.ItalianAnalyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.IntField;
import org.apache.lucene.document.LongField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.index.Term;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.maven.plugin.MojoFailureException;
import org.apache.maven.plugin.logging.Log;
import org.apache.tika.exception.TikaException;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.AutoDetectParser;
import org.apache.tika.sax.BodyContentHandler;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Element;
import org.jsoup.nodes.Node;
import org.jsoup.parser.Parser;
import org.jsoup.select.Elements;
import org.xml.sax.SAXException;

import docet.engine.model.FaqEntry;
import docet.engine.model.TOC;

/**
 *
 * @author matteo.casadei
 *
 */
public final class DocetPluginUtils {

    public static final String FAQ_DEFAULT_PAGE_PREFIX = "faq_";
    // no. of chars to consider when extracting a sort of abstract to shown
    // later on search results.
    public static final int SHORT_SEARCH_TEXT_DEFAULT_LENGTH = 300;
    public static final int SHORT_SEARCH_ANSWER_TEXT_DEFAULT_LENGTH = 90;
    public static final String FAQ_TOC_ID = "docet-faq-menu";
    public static final String FAQ_HOME_ANCHOR_ID = "docet-faq-main-link";

    private static final String CONFIG_NAMES_FOLDER_PAGES = "pages";
    private static final String CONFIG_NAMES_FOLDER_PDFS = "pdf";
    private static final String CONFIG_NAMES_FOLDER_IMAGES = "imgs";
    private static final String CONFIG_NAMES_FILE_TOC = "toc.html";
    private static final String DOCET_HTML_ATTR_REFERENCE_LANGUAGE_NAME = "reference-language";

    private static final String ENCODING_UTF8 = "UTF-8";

    private static final int INDEX_DOCTYPE_PAGE = 1;
    private static final int INDEX_DOCTYPE_FAQ = 2;

    public enum Language {
        EN, FR, IT;

        @Override
        public String toString() {
            return super.toString().toLowerCase();
        }

        public static Language getLanguageByCode(final String code) {
            final List<Language> foundLangs = Arrays.asList(Language.values()).stream().filter(l -> l.toString().equals(code))
                .collect(Collectors.toList());
            if (foundLangs.isEmpty()) {
                return null;
            } else {
                return foundLangs.get(0);
            }
        }
    }

    private DocetPluginUtils() {
    }

    public static Map<Language, List<DocetIssue>> validateDocs(final Path srcDir, final Map<Language, List<FaqEntry>> faqs, final Log log)
        throws MojoFailureException {
        Map<Language, List<DocetIssue>> result = new EnumMap<>(Language.class);
        // checking referred pages for each doc
        for (Language lang : Language.values()) {
            Path langPath = srcDir.resolve(lang.toString());
            langPath = langPath.resolve(CONFIG_NAMES_FOLDER_PAGES);
            if (Files.exists(langPath) && Files.isDirectory(langPath)) {
                List<FaqEntry> entriesForLang = new ArrayList<>();
                faqs.put(lang, entriesForLang);
                int indexed = validateDocsForLanguage(langPath, lang, entriesForLang, (severity, msg) -> {
                    final List<DocetIssue> messages = new ArrayList<>();
                    messages.add(new DocetIssue(severity, msg));
                    result.merge(lang, messages, (l1, l2) -> {
                        l1.addAll(l2);
                        return l1;
                    });
                }, log);
                if (indexed == 0) {
                    final List<DocetIssue> messages = new ArrayList<>();
                    messages.add(new DocetIssue(Severity.WARN, "No docs found for language"));
                    result.put(lang, messages);
                }
            } else {
                final List<DocetIssue> messages = new ArrayList<>();
                messages.add(new DocetIssue(Severity.WARN, "No folder found for language"));
                result.put(lang, messages);
            }
        }
        return result;
    }

    public static Map<Language, List<DocetIssue>> validatePdfs(final Path srcDir, final Log log)
        throws MojoFailureException {
        Map<Language, List<DocetIssue>> result = new EnumMap<>(Language.class);
        // checking referred pages for each doc
        for (Language lang : Language.values()) {
            Path langPath = srcDir.resolve(lang.toString());
            langPath = langPath.resolve(CONFIG_NAMES_FOLDER_PDFS);
            if (Files.exists(langPath) && Files.isDirectory(langPath)) {
                int indexed = validatePdfsForLanguage(langPath, lang, (severity, msg) -> {
                    final List<DocetIssue> messages = new ArrayList<>();
                    messages.add(new DocetIssue(severity, msg));
                    result.merge(lang, messages, (l1, l2) -> {
                        l1.addAll(l2);
                        return l1;
                    });
                }, log);
                if (indexed == 0) {
                    final List<DocetIssue> messages = new ArrayList<>();
                    messages.add(new DocetIssue(Severity.WARN, "No pdfs found for language"));
                    result.put(lang, messages);
                }
            } else {
                final List<DocetIssue> messages = new ArrayList<>();
                messages.add(new DocetIssue(Severity.WARN, "No pdfs found for language"));
                result.put(lang, messages);
            }
        }
        return result;
    }

    private static Set<String> retrieveImageNames(final Path imgsFolder, final BiConsumer<Severity, String> call,
        final Log log) throws IOException {
        final Set<String> res = new HashSet<>();

        if (!Files.isDirectory(imgsFolder)) {
            call.accept(Severity.WARN, "[IMGS] Directory " + imgsFolder.toAbsolutePath() + " not found");
            return res;
        }

        Files.walkFileTree(imgsFolder, new SimpleFileVisitor<Path>() {
            @Override
            public FileVisitResult preVisitDirectory(final Path file, final BasicFileAttributes attrs) throws IOException {
                if (file.toFile().getName().startsWith(".")) {
                    return FileVisitResult.SKIP_SUBTREE;
                }
                return FileVisitResult.CONTINUE;
            }

            @Override
            public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException {
                if (file.getFileName().toString().startsWith(".")) {
                    return FileVisitResult.CONTINUE;
                }
                final String fileName = file.toFile().getName();
                if (log.isDebugEnabled()) {
                    log.debug("[retrieveImageNames] add image file " + fileName + " to set of found images");
                }
                res.add(fileName);
                return FileVisitResult.CONTINUE;
            }
        });
        return res;
    }

    public static int validatePdfsForLanguage(final Path path, final Language lang,
        final BiConsumer<Severity, String> call, final Log log) throws MojoFailureException {
        final Holder<Integer> scannedDocs = new Holder<>(0);
        try {
            Files.walkFileTree(path, new SimpleFileVisitor<Path>() {
                @Override
                public FileVisitResult preVisitDirectory(final Path file, final BasicFileAttributes attrs)
                    throws IOException {
                    if (file.toFile().getName().startsWith(".")) {
                        return FileVisitResult.SKIP_SUBTREE;
                    }
                    return FileVisitResult.CONTINUE;
                }

                @Override
                public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException {
                    if (file.getFileName().toString().startsWith(".")) {
                        return FileVisitResult.CONTINUE;
                    }
                    try {
                        validatePdf(file, call);
                        scannedDocs.setValue(scannedDocs.value + 1);
                    } catch (Exception ex) {
                        call.accept(Severity.ERROR, "File " + file + " cannot be read. " + ex);
                    }
                    return FileVisitResult.CONTINUE;
                }
            });
            return scannedDocs.value;
        } catch (Exception e) {
            throw new MojoFailureException("Failure while visiting source docs.", e);
        }
    }

    public static int validateDocsForLanguage(final Path path, final Language lang, final List<FaqEntry> faqs,
        final BiConsumer<Severity, String> call, final Log log) throws MojoFailureException {
        final Holder<Integer> scannedDocs = new Holder<>(0);
        final Holder<Boolean> mainPageFound = new Holder<>(false);
        // the following Map is used to check duplicated titles
        final Map<String, List<String>> titleInPages = new HashMap<>();
        final Map<String, Integer> filesCount = new HashMap<>();
        final Map<String, String> foundFaqPages = new HashMap<>();
        try {
            final Path toc = path.getParent().resolve(CONFIG_NAMES_FILE_TOC);
            final Map<String, String> linkedPagesInToc = new HashMap<>();
            if (Files.exists(toc)) {
                validateToc(toc, linkedPagesInToc, call);
                validateFaqIndex(toc, foundFaqPages, linkedPagesInToc, call);
            } else {
                call.accept(Severity.ERROR, "TOC 'Table of Contents' file 'toc.html' not found!");
            }

            final Path imgs = path.getParent().resolve(CONFIG_NAMES_FOLDER_IMAGES);
            final Set<String> foundImages = retrieveImageNames(imgs, call, log);
            final Set<String> imagesLinkedInPages = new HashSet<>();

            if (log.isDebugEnabled()) {
                linkedPagesInToc.keySet().stream().forEach(item -> {
                    log.debug("[" + lang + "] LINKED PAGE FOUND -> '" + item + "'");
                });
            }

            Files.walkFileTree(path, new SimpleFileVisitor<Path>() {
                @Override
                public FileVisitResult preVisitDirectory(final Path file, final BasicFileAttributes attrs) throws IOException {
                    if (file.toFile().getName().startsWith(".")) {
                        return FileVisitResult.SKIP_SUBTREE;
                    }
                    return FileVisitResult.CONTINUE;
                }

                @Override
                public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException {
                    if (file.getFileName().toString().startsWith(".")) {
                        return FileVisitResult.CONTINUE;
                    }
                    if (file.endsWith("main.html")) {
                        mainPageFound.setValue(true);
                        linkedPagesInToc.put("main.html", "Main page");
                    }
                    try {
                        final String fileName = file.toFile().getName();
                        if (linkedPagesInToc.get(fileName) != null) {
                            validateDoc(path, file, call, titleInPages, imagesLinkedInPages, filesCount);
                            scannedDocs.setValue(scannedDocs.getValue() + 1);
                        } else {
                            call.accept(Severity.ERROR, "NON-LINKED (UNUSED) FILE FOUND: " + fileName);
                        }
                    } catch (Exception ex) {
                        call.accept(Severity.ERROR, "File " + file + " cannot be read. " + ex);
                    }
                    return FileVisitResult.CONTINUE;
                }
            });
            checkForDuplicatePageTitles(titleInPages, call);
            checkForDuplicateFileNames(filesCount, call);
            checkForOrphanImages(foundImages, imagesLinkedInPages, call);
            final Path faqPath = path.getParent().resolve("faq");
            validateFaqs(faqPath, faqs, call, foundFaqPages, linkedPagesInToc, false);
            validateFaqs(faqPath, faqs, call, foundFaqPages, linkedPagesInToc, true);
            checkForOrphanFaqLinks(faqs, foundFaqPages.keySet(), call);
            if (!mainPageFound.getValue()) {
                call.accept(Severity.WARN, "Main page file 'main.html' not found");
            }

            linkedPagesInToc.keySet().stream().filter(pageName -> pageName.startsWith(FAQ_DEFAULT_PAGE_PREFIX))
                .forEach(pageName -> {
                    call.accept(Severity.ERROR, "NON-LINKED (UNUSED) FAQ FILE FOUND: " + pageName);
                });
        } catch (Exception e) {
            throw new MojoFailureException("Failure while visiting source docs.", e);
        }
        return scannedDocs.getValue();
    }

    /**
     * Checks whether an image in file system is linked from at least one page within the source page files.
     *
     * @param imgsInFileSystem the set of images found on the doc package file system
     * @param imgsLinked the set of images linked from source page files
     * @param issueLogger consumer used to keep track of issues found on the aforemetioned checks
     */
    private static void checkForOrphanImages(final Set<String> imgsInFileSystem, final Set<String> imgsLinked,
        final BiConsumer<Severity, String> issueLogger) {
        final Set<String> linkedImgsNames = imgsLinked.stream()
            .map(img -> img.split("@")[0])
            .collect(Collectors.toSet());

        //check for unused images
        imgsInFileSystem.stream()
            .filter(img -> !linkedImgsNames.contains(img))
            .forEach(img -> issueLogger.accept(Severity.ERROR, "[ORPHAN IMAGE] Found UNUSED IMAGE " + img));
    }

    private static void checkForOrphanFaqLinks(final List<FaqEntry> faqsToAdd, final Set<String> faqsWithLinks,
        final BiConsumer<Severity, String> issueCall) {
        //checking that all the faqslinks point to faq pages present in the list of faqentry to be included in the doc
        //package
        final List<String> faqsToAddNames = faqsToAdd.stream().map(entry -> entry.getFaqPath().toFile().getName())
            .collect(Collectors.toList());
        faqsWithLinks.stream().filter(faq -> !faqsToAddNames.contains(faq.split("@")[0]))
            .collect(Collectors.toList()).forEach(orphan -> {
            final String[] tokens = orphan.split("@");
            final String srcPage = tokens[1];
            final String targetFaqPage = tokens[0];
            issueCall.accept(Severity.ERROR, "[FAQ] [ORPHAN LINK] [" + srcPage + "] links to unexisting faq -> " + targetFaqPage);
        });
    }

    private static void validateFaqs(final Path faqFolderPath, final List<FaqEntry> faqs, final BiConsumer<Severity, String> call, final Map<String, String> faqPages, final Map<String, String> pagesFoundInTOC, boolean generateEntries)
        throws IOException {
        if (!Files.isDirectory(faqFolderPath)) {
            call.accept(Severity.WARN, "[FAQ] Directory " + faqFolderPath.toAbsolutePath() + " not found");
            return;
        }
        Files.walkFileTree(faqFolderPath, new SimpleFileVisitor<Path>() {
            @Override
            public FileVisitResult preVisitDirectory(final Path file, final BasicFileAttributes attrs) throws IOException {
                if (file.toFile().getName().startsWith(".")) {
                    return FileVisitResult.SKIP_SUBTREE;
                }
                return FileVisitResult.CONTINUE;
            }

            @Override
            public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException {
                if (file.getFileName().toString().startsWith(".")) {
                    return FileVisitResult.CONTINUE;
                }
                final File fileToParse = file.toFile();
                try (InputStream stream = Files.newInputStream(file)) {
                    final org.jsoup.nodes.Document htmlDoc = Jsoup.parseBodyFragment(FileUtils.readFileToString(fileToParse, ENCODING_UTF8));

                    // checking linked pages exists
                    Elements links = htmlDoc.select("a.faq-link");
                    links.stream().forEach(link -> {
                        faqPages.put(link.attr("href").split("#")[0] + "@" + fileToParse.getName(), link.text());
                    });
                }
                if (generateEntries) {
                    //if this is a first-level faq the corresponding name should be found in toc
                    //if so delete the corresponding name from the set so that is not be counted as not found 1st-level
                    //faq
                    final String title = pagesFoundInTOC.remove(FAQ_DEFAULT_PAGE_PREFIX + fileToParse.getName());
                    final boolean found = faqPages.keySet().stream().anyMatch(faq -> faq.startsWith(fileToParse.getName() + "@"));
                    if (found || title != null) {
                        final String faqTitle;
                        if (title != null) {
                            faqTitle = title;
                        } else {
                            faqTitle = faqPages.entrySet().stream()
                                .filter(e -> e.getKey().startsWith(fileToParse.getName()))
                                .findFirst().map(e -> e.getValue()).orElse("");
                        }
                        try {
                            parseFaqEntry(file, faqTitle, faqs, call);
                        } catch (Exception ex) {
                            call.accept(Severity.WARN, "FAQ File " + file + " cannot be read. " + ex);
                        }
                    } else {
                        call.accept(Severity.ERROR, "[FAQ] Found an unused faq file '" + file.toFile().getName() + "'");
                    }
                }
                return FileVisitResult.CONTINUE;
            }
        });
    }

    private static void parseFaqEntry(final Path file, final String faqTitle, final List<FaqEntry> faqs,
        final BiConsumer<Severity, String> call) throws IOException, SAXException, TikaException {
        // Faq page validation
        final Path pagesPath = file.getParent().getParent().resolve(CONFIG_NAMES_FOLDER_PAGES);;
        final Path pdfPath = pagesPath.getParent().resolve(CONFIG_NAMES_FOLDER_PDFS);
        final Path imagesPath = pagesPath.getParent().resolve("imgs");
        final Path faqPath = pagesPath.getParent().resolve("faq");
        try (InputStream stream = Files.newInputStream(file)) {
            final org.jsoup.nodes.Document htmlDoc = Jsoup.parseBodyFragment(FileUtils.readFileToString(file.toFile(), ENCODING_UTF8));

            // checking overall doc structure
            final Elements divMain = htmlDoc.select("div#main");
            switch (divMain.size()) {
                case 0:
                    call.accept(Severity.ERROR, "[FAQ] [" + file.getFileName() + "] Body is empty");
                    break;
                case 1:
                    break;
                default:
                    call.accept(Severity.ERROR,
                        "[FAQ] [" + file.getFileName() + "] Expected just one main element <div id=\"main\">, found: " + divMain.size());
            }

            // checking linked pages exists
            Elements links = htmlDoc.select("a:not(.question)");
            links.stream().forEach(link -> {
                if (!link.hasAttr("href")) {
                    call.accept(Severity.ERROR,
                        "[FAQ] [" + file.getFileName() + "] Found anchor without HREF attribute '" + link + "'");
                    return;
                }
                final String href = link.attr("href");
                final boolean isFaqLink = link.hasClass("faq-link");
                final boolean pageExists;
                final boolean isPdfLink;
                String pageLink = null;
                if (href.startsWith("#")) {
                    return;
                }
                try {
                    if (href.startsWith("http://") || href.startsWith("https://")) {
                        // no going to check an alleged external link
                        pageExists = true;
                        pageLink = href;
                        isPdfLink = false;
                    } else {
                        final String[] linkTokens = link.attr("href").split("/");
                        pageLink = linkTokens[linkTokens.length - 1];
                        if (pageLink.startsWith("#")) {
                            pageExists = !htmlDoc.select(pageLink).isEmpty();
                            isPdfLink = false;
                        } else if (pageLink.isEmpty()) {
                            call.accept(Severity.WARN,
                                "[FAQ] [" + file.getFileName() + "] Found anchor '" + link + "' WITH EMPTY HREF:" + " was this done on purpose?");
                            pageExists = true;
                            isPdfLink = false;
                        } else if (pageLink.endsWith(".pdf")) {
                            isPdfLink = true;
                            pageExists = fileExists(pdfPath, pageLink.replace(".pdf", ".html"));
                        } else {
                            isPdfLink = false;
                            pageExists = fileExists((isFaqLink ? faqPath : pagesPath), pageLink);
                        }
                    }
                    if (!pageExists) {
                        if (isPdfLink) {
                            call.accept(Severity.ERROR, "[FAQ] [" + file.getFileName() + "] Referred pdf '" + pageLink + "' does not exist");
                        } else {
                            call.accept(Severity.ERROR, "[FAQ] [" + file.getFileName() + "] Referred " + (isFaqLink ? "faq" : "") + " page '" + pageLink + "' does not exist");
                        }
                    }
                } catch (IOException e) {
                    call.accept(Severity.ERROR,
                        "[FAQ] [" + file.getFileName() + "] Referred page '" + pageLink + "' existence cannot be checked. Reason: " + e);
                }
            });

            // checking referred images exists
            Elements images = htmlDoc.getElementsByTag("img");
            images.stream().forEach(image -> {
                if (!image.hasAttr("src")) {
                    call.accept(Severity.ERROR,
                        "[FAQ] [" + file.getFileName() + "] Found image without SRC attribute '" + image + "'");
                    return;
                }
                final String[] linkTokens = image.attr("src").split("/");
                final String imageLink = linkTokens[linkTokens.length - 1];
                final String fileExtension = imageLink.substring(imageLink.lastIndexOf('.') + 1);
                final boolean formatAllowed = allowedFileExtension(fileExtension);
                try {
                    if (formatAllowed) {
                        final boolean imageExists = fileExists(imagesPath, imageLink);
                        if (!imageExists) {
                            call.accept(Severity.ERROR, "[FAQ] [" + file.getFileName() + "] Referred image '" + imageLink + "' does not exist");
                        }
                    } else {
                        call.accept(Severity.ERROR, "[FAQ] [" + file.getFileName() + "] Referred image '" + imageLink + "' unsupported format");
                    }
                } catch (IOException e) {
                    call.accept(Severity.ERROR,
                        "[FAQ] [" + file.getFileName() + "] Referred image '" + imageLink + "' existence cannot be inferred. Reason: " + e);
                }
            });

            final Elements title = htmlDoc.select("h1");
            final long foundTitles = title.stream().peek(h1 -> {
                if (h1.text().isEmpty()) {
                    call.accept(Severity.ERROR, "[FAQ] [" + file.getFileName() + "] Title is empty");
                }
            }).count();
            if (foundTitles > 1) {
                call.accept(Severity.ERROR, "[FAQ] [" + file.getFileName() + "] Found " + foundTitles + ". Just one should be provided");
            }
            if (foundTitles == 0) {
                call.accept(Severity.ERROR, "[FAQ] [" + file.getFileName() + "] No titles Found: 1 must be provided");
            }

            // checking divs
            Elements divBoxes = htmlDoc.select("div");
            divBoxes.stream().forEach(div -> {
                // check for box div
                if (div.hasClass("msg")
                    && (!(div.hasClass("tip") || div.hasClass("info") || div.hasClass("note") || div.hasClass("warning")))) {
                    call.accept(Severity.ERROR, "[FAQ] [" + file.getFileName() + "] Found div.msg with not qualifying class [tip|info|note|warning]");
                }
            });
        }
        // add found entry to the list of pages to add to faq (on indexing/zipping stage)
        faqs.add(new FaqEntry(file, faqTitle));
    }

    private static void validateDoc(final Path rootPath, final Path file, final BiConsumer<Severity, String> call,
        final Map<String, List<String>> titleInPages, final Set<String> linkedImages,
        final Map<String, Integer> filesCount)
        throws IOException, SAXException, TikaException {
        final Path pagesPath = rootPath;
        final Path pdfPath = rootPath.getParent().resolve(CONFIG_NAMES_FOLDER_PDFS);
        final Path imagesPath = rootPath.getParent().resolve("imgs");
        final Path faqPath = rootPath.getParent().resolve("faq");
        try (InputStream stream = Files.newInputStream(file)) {
            final String fileName = file.getFileName().toString();
            filesCount.merge(fileName, Integer.valueOf(1), (c1, c2) -> c1 + c2);
            final org.jsoup.nodes.Document htmlDoc = Jsoup.parseBodyFragment(FileUtils.readFileToString(file.toFile(), ENCODING_UTF8));

            // checking overall doc structure
            final Elements divMain = htmlDoc.select("div#main");
            switch (divMain.size()) {
                case 0:
                    call.accept(Severity.ERROR, "[" + file.getFileName() + "] Body is empty");
                    break;
                case 1:
                    break;
                default:
                    call.accept(Severity.ERROR,
                        "[" + file.getFileName() + "] Expected just one main element <div id=\"main\">, found: " + divMain.size());
            }

            // checking linked pages exists
            Elements links = htmlDoc.select("a:not(.question)");
            links.stream().forEach(link -> {
                if (!link.hasAttr("href")) {
                    call.accept(Severity.ERROR,
                        "[" + file.getFileName() + "] Found anchor without HREF attribute '" + link + "'");
                    return;
                }
                final String href = link.attr("href");
                final boolean isFaqLink = link.hasClass("faq-link");
                final boolean pageExists;
                final boolean isPdfLink;
                String pageLink = null;
                if (href.startsWith("#")) {
                    return;
                }
                try {
                    if (href.startsWith("http://") || href.startsWith("https://")) {
                        // no going to check an alleged external link
                        pageExists = true;
                        pageLink = href;
                        isPdfLink = false;
                    } else {
                        final String[] linkTokens = link.attr("href").split("/");
                        pageLink = linkTokens[linkTokens.length - 1];
                        if (pageLink.startsWith("#")) {
                            pageExists = !htmlDoc.select(pageLink).isEmpty();
                            isPdfLink = false;
                        } else if (pageLink.isEmpty()) {
                            call.accept(Severity.WARN,
                                "[" + file.getFileName() + "] Found anchor '" + link + "' WITH EMPTY HREF:" + " was this done on purpose?");
                            pageExists = true;
                            isPdfLink = false;
                        } else if (pageLink.endsWith(".pdf")) {
                            isPdfLink = true;
                            pageExists = fileExists(pdfPath, pageLink.replace(".pdf", ".html"));
                        } else {
                            isPdfLink = false;
                            pageExists = fileExists((isFaqLink ? faqPath : pagesPath), pageLink);
                        }
                    }
                    if (!pageExists) {
                        if (isPdfLink) {
                            call.accept(Severity.ERROR, "[" + file.getFileName() + "] Referred pdf '" + pageLink + "' does not exist");
                        } else {
                            call.accept(Severity.ERROR, "[" + file.getFileName() + "] Referred " + (isFaqLink ? "faq" : "") + " page '" + pageLink + "' does not exist");
                        }
                    }
                } catch (IOException e) {
                    call.accept(Severity.ERROR,
                        "[" + file.getFileName() + "] Referred page '" + pageLink + "' existence cannot be checked. Reason: " + e);
                }
            });

            // checking referred images exists
            Elements images = htmlDoc.getElementsByTag("img");
            images.stream().forEach(image -> {
                if (!image.hasAttr("src")) {
                    call.accept(Severity.ERROR,
                        "[" + file.getFileName() + "] Found image without SRC attribute '" + image + "'");
                    return;
                }
                final String[] linkTokens = image.attr("src").split("/");
                final String imageLink = linkTokens[linkTokens.length - 1];
                linkedImages.add(imageLink + "@" + fileName);
                final String fileExtension = imageLink.substring(imageLink.lastIndexOf('.') + 1);
                final boolean formatAllowed = allowedFileExtension(fileExtension);
                try {
                    if (formatAllowed) {
                        final boolean imageExists = fileExists(imagesPath, imageLink);
                        if (!imageExists) {
                            call.accept(Severity.ERROR, "[" + file.getFileName() + "] Referred image '" + imageLink + "' does not exist");
                        }
                    } else {
                        call.accept(Severity.ERROR, "[" + file.getFileName() + "] Referred image '" + imageLink + "' unsupported format");
                    }
                } catch (IOException e) {
                    call.accept(Severity.ERROR,
                        "[" + file.getFileName() + "] Referred image '" + imageLink + "' existence cannot be inferred. Reason: " + e);
                }
            });

            final Elements title = htmlDoc.select("h1");
            final long foundTitles = title.stream().peek(h1 -> {
                if (h1.text().isEmpty()) {
                    call.accept(Severity.ERROR, "[" + file.getFileName() + "] Title is empty");
                }
            }).count();
            if (foundTitles > 1) {
                call.accept(Severity.ERROR, "[" + file.getFileName() + "] Found " + foundTitles + ". Just one should be provided");
            }
            if (foundTitles == 0) {
                call.accept(Severity.ERROR, "[" + file.getFileName() + "] No titles Found: 1 must be provided");
            } else if (foundTitles == 1) {
                final List<String> currentPageAsList = new ArrayList<>();
                currentPageAsList.add(file.getFileName().toString());
                // check if title has been already used on another page
                titleInPages.merge(title.text().trim(), currentPageAsList, (l1, l2) -> {
                    l1.addAll(l2);
                    return l1;
                });
            }

            // checking divs
            Elements divBoxes = htmlDoc.select("div");
            divBoxes.stream().forEach(div -> {
                // check for box div
                if (div.hasClass("msg")
                    && (!(div.hasClass("tip") || div.hasClass("info") || div.hasClass("note") || div.hasClass("warning")))) {
                    call.accept(Severity.ERROR, "[" + file.getFileName() + "] Found div.msg with not qualifying class [tip|info|note|warning]");
                }
            });
        }
    }

    private static void checkForDuplicatePageTitles(final Map<String, List<String>> titleInPages, final BiConsumer<Severity, String> call) {
        titleInPages.entrySet().stream().forEach(entry -> {
            if (entry.getValue().size() > 1) {
                call.accept(Severity.ERROR, "Title '" + entry.getKey() + "' has been used on multiple pages: "
                    + Arrays.toString(entry.getValue().toArray(new String[]{})));
            }
        });
    }

    private static void checkForDuplicateFileNames(final Map<String, Integer> pageNameCount, final BiConsumer<Severity, String> call) {
        pageNameCount.entrySet().stream().forEach(entry -> {
            if (entry.getValue() > 1) {
                call.accept(Severity.ERROR,
                    "[" + entry.getKey() + "] found " + entry.getValue() + " instances: this is going to cause linking issues!");
            }
        });
    }

    private static void validateFaqIndex(final Path faq, final Map<String, String> foundFaqs,
        final Map<String, String> foundLinkedFaqs, final BiConsumer<Severity, String> call)
        throws IOException {
        try (InputStream stream = Files.newInputStream(faq)) {
            final org.jsoup.nodes.Document htmlDoc = Jsoup.parseBodyFragment(FileUtils.readFileToString(faq.toFile(), ENCODING_UTF8));
            final Elements faqItems = htmlDoc.select("#" + FAQ_TOC_ID + " a");
            if (faqItems.isEmpty()) {
                call.accept(Severity.WARN, "[TOC] NO FAQs defined in table of contents");
                return;
            }
            faqItems.forEach(item -> {
                final String faqHref = item.attr("href").split("#")[0];
                final Path faqItemFile = faq.getParent().resolve("faq" + File.separator + faqHref);
                if (!faqItemFile.toFile().exists()) {
                    call.accept(Severity.ERROR, "[FAQ] A file '" + faqHref + "' is linked in faq list but does not exist");
                } else {
                    foundFaqs.put(faqHref + "@" + faq.toFile().getName(), item.text());
                    foundLinkedFaqs.put(FAQ_DEFAULT_PAGE_PREFIX + faqHref.split("#")[0],
                        item.text().trim());
                }
            });
        }
    }

    private static void validatePdf(final Path pdfToc, final BiConsumer<Severity, String> call)
        throws IOException, SAXException {
        final String pdfName = pdfToc.toFile().getName();
        try (InputStream stream = Files.newInputStream(pdfToc)) {
            final org.jsoup.nodes.Document pdfDoc = Jsoup.parse(FileUtils.readFileToString(pdfToc.toFile(), ENCODING_UTF8));
            //check a title has been defined
            final Element title = pdfDoc.getElementsByTag("title").first();
            if (title != null) {
                if (title.text().isEmpty()) {
                    call.accept(Severity.ERROR, "[PDF] [" + pdfName + "] Title cannot be empty");
                }
            } else {
                call.accept(Severity.ERROR, "[PDF] [" + pdfName + "] No valid title was found");
            }

            //checking bare pdf's toc structure
            Elements nav = pdfDoc.getElementsByTag("nav");
            checkNavStructure(nav, "[PDF] [" + pdfName + "]", call);

            //check linked pages actually do exist
            final Path pagesPathForLang = pdfToc.getParent().getParent().resolve(CONFIG_NAMES_FOLDER_PAGES);
            final Path mainDocsFolder = pdfToc.getParent().getParent().getParent();
            Elements links = pdfDoc.getElementsByTag("a");
            links.stream().forEach(link -> {
                final String[] linkTokens = link.attr("href").split("/");
                final String pageLink = linkTokens[linkTokens.length - 1];
                boolean pageExists = false;
                final Path pagesCheckFolder;
                final String refLanguage = link.attr(DOCET_HTML_ATTR_REFERENCE_LANGUAGE_NAME);
                if (refLanguage.isEmpty()) {
                    pagesCheckFolder = pagesPathForLang;
                } else {
                    final Language validlang = Language.getLanguageByCode(refLanguage);
                    if (validlang == null) {
                        call.accept(Severity.ERROR, "[PDF] [" + pdfName + "] page '" + pageLink + "' reference language '"
                            + refLanguage + "' is not supported!");
                        return;
                    }
                    pagesCheckFolder = mainDocsFolder.resolve(refLanguage);
                }
                try {
                    pageExists = fileExists(pagesCheckFolder, pageLink);
                    if (!pageExists) {
                        final String referenceLangMsg;
                        if (refLanguage.isEmpty()) {
                            referenceLangMsg = "";
                        } else {
                            referenceLangMsg = " (reference language=" + refLanguage + ")";
                        }
                        call.accept(Severity.ERROR, "[PDF] [" + pdfName + "] page '" + pageLink + "'"
                            + referenceLangMsg + " does not exist");
                    }
                } catch (IOException e) {
                    call.accept(Severity.ERROR, "[PDF] [" + pdfName + "] page '" + pageLink + "' existence cannot be checked. Reason: " + e);
                }
            });
        }
    }

    private static void validateToc(final Path toc, final Map<String, String> linkedPagesFound, BiConsumer<Severity, String> call)
        throws IOException, SAXException {
        try (InputStream stream = Files.newInputStream(toc)) {
            final org.jsoup.nodes.Document htmlDoc = Jsoup.parseBodyFragment(FileUtils.readFileToString(toc.toFile(), ENCODING_UTF8));

            // check structure
            Elements nav = htmlDoc.getElementsByTag("nav");
            checkNavStructure(nav, "[TOC]", call);

            final Set<String> linkedPages = new HashSet<>();
            // checking linked pages exists
            final Path pagesPath = toc.getParent().resolve(CONFIG_NAMES_FOLDER_PAGES);
            final Path faqPath = toc.getParent().resolve("faq");
            Elements links = htmlDoc.getElementsByTag("a");

            Element faqMainLink = htmlDoc.getElementById(FAQ_HOME_ANCHOR_ID);
            Elements faqAnchors = new Elements();
            if (faqMainLink != null) {
                faqAnchors = faqMainLink.parent().select("ul");
                if (!faqAnchors.isEmpty()) {
                    faqAnchors = faqAnchors.select("a");
                }
            }
            final Elements faqUltimateAnchors = faqAnchors;
            links.stream().forEach(link -> {
                final String[] linkTokens = link.attr("href").split("/");
                final String pageLink = linkTokens[linkTokens.length - 1];
                boolean pageExists = false;
                boolean pageAlreadyLinked = false;
                final boolean isFaq = faqUltimateAnchors.contains(link);
                try {
                    pageExists = fileExists(isFaq ? faqPath : pagesPath, pageLink);
                    if (!pageExists) {
                        call.accept(Severity.ERROR, "[TOC] Referred " + (isFaq ? "FAQ" : "") + " page '" + pageLink + "' does not exist");
                    } else {
                        pageAlreadyLinked = !linkedPages.add(isFaq + "-" + pageLink);
                        if (isFaq) {
                            linkedPagesFound.put(FAQ_DEFAULT_PAGE_PREFIX + pageLink.split("#")[0], link.text().trim());
                        } else {
                            linkedPagesFound.put(pageLink.split("#")[0], link.text().trim());
                        }
                    }
                    if (pageAlreadyLinked) {
                        call.accept(Severity.ERROR, "[TOC] " + (isFaq ? "FAQ" : "") + " page '" + pageLink + "' is mentioned in TOC multiple times");
                    }
                } catch (IOException e) {
                    call.accept(Severity.ERROR, "[TOC] Referred " + (isFaq ? "FAQ" : "") + " page '" + pageLink + "' existence cannot be checked. Reason: " + e);
                }
            });
        }
    }

    private static void checkNavStructure(final Elements nav, final String logPrefix, BiConsumer<Severity, String> call) {
        final int navNum = nav.size();
        if (navNum == 0) {
            call.accept(Severity.ERROR, logPrefix + " is currently empty");
        }
        if (navNum > 1) {
            call.accept(Severity.ERROR, logPrefix + " defined " + navNum + " navs. One expected.");
        }
        nav.stream().forEach(n -> {
            final Elements ul = n.getElementsByTag("ul");
            final int ulNum = ul.size();
            if (ulNum == 0) {
                call.accept(Severity.ERROR, logPrefix + " is currently empty");
            }
            final List<Node> straightChildUls = n.childNodes().stream().filter(nd -> "ul".equals(nd.nodeName()))
                .collect(Collectors.toList());
            if (straightChildUls.size() > 1) {
                call.accept(Severity.ERROR, logPrefix + " nav contains " + ulNum + " ULs. One expected.");
            }
            if (ulNum == 1) {
                final Elements lis = ul.get(0).children();
                if (lis.isEmpty()) {
                    call.accept(Severity.ERROR, logPrefix + " is currently empty");
                }
                lis.stream().forEach(l -> {
                    if (!"li".equals(l.tag().toString())) {
                        call.accept(Severity.ERROR, logPrefix + " Expected <li>, found: <" + l.tag() + ">");
                    } else {
                        final Elements anchor = l.children();
                        switch (anchor.size()) {
                            case 0:
                                call.accept(Severity.ERROR, logPrefix + " Empty <li> found");
                                break;
                            case 1:
                                if (!"a".equals(anchor.get(0).tag().toString())) {
                                    call.accept(Severity.ERROR, logPrefix + " found <li> with no <a> included");
                                }
                                break;
                            case 2:
                                if (!"a".equals(anchor.get(0).tag().toString())) {
                                    call.accept(Severity.ERROR, logPrefix + " <li> -> expected <a>, found <" + anchor.get(0).tag() + ">");
                                }
                                if (!"ul".equals(anchor.get(1).tag().toString())) {
                                    call.accept(Severity.ERROR, logPrefix + " <li> -> expected <ul>, found <" + anchor.get(1).tag() + ">");
                                }
                                break;
                            default:
                                call.accept(Severity.ERROR, logPrefix + " found <li> containing " + anchor.size() + " elements."
                                    + " An <li> must include no more than a <a> and a <ul>!");
                        }
                    }
                });
            }
        });
    }

    private static boolean fileExists(final Path basePath, final String fileName) throws IOException {
        final Holder<Boolean> result = new Holder<>(false);
        final String actualFileName;
        if (fileName.contains("#")) {
            actualFileName = fileName.split("#")[0];
        } else {
            actualFileName = fileName;
        }
        Files.walkFileTree(basePath, new SimpleFileVisitor<Path>() {
            @Override
            public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException {
                if (file.endsWith(actualFileName)) {
                    result.setValue(true);
                    return FileVisitResult.TERMINATE;
                }
                return FileVisitResult.CONTINUE;
            }
        });
        return result.value;
    }

    private static boolean allowedFileExtension(final String fileExtension) {
        return Arrays.asList(ForbiddenExtensions.values()).stream().filter(ext -> ext.extension().equals(fileExtension)).count() == 0;
    }

    private enum ForbiddenExtensions {
        JPEG("jpeg"), JPG("jpg"), GIF("gif");

        private String extension;

        private ForbiddenExtensions(final String extension) {
            this.extension = extension;
        }

        public String extension() {
            return this.extension;
        }
    }

    public static void copyingDocs(final Path outDir, final Log log) {
        log.info("Operation will be available soon!");
        throw new UnsupportedOperationException("Operation not yet available");
    }

    public static Map<Language, List<DocetIssue>> generatePdfsForLanguage(final Path srcDir, final Path outDir, final Path tmpDir,
        final String langCode, final Log log) throws MojoFailureException {
        final Map<Language, List<DocetIssue>> result = new EnumMap<>(Language.class);
        final List<DocetIssue> messages = new ArrayList<>();
        Path langPath = srcDir.resolve(langCode);
        langPath = langPath.resolve(CONFIG_NAMES_FOLDER_PAGES);
        final Language lang = Language.getLanguageByCode(langCode);
        if (lang == null) {
            throw new MojoFailureException("Language [" + langCode + "] is not supported!");
        }
        if (Files.exists(langPath) && Files.isDirectory(langPath)) {
            final Path tocPath = langPath.getParent().resolve(CONFIG_NAMES_FILE_TOC);
            final Path outputDir = outDir;
            TOC comprenhesiveToc;
            try {
                comprenhesiveToc = parseTOCFromPath(tocPath, tmpDir, messages);
                comprenhesiveToc.getItems().stream().forEach(item -> {
                    String outFileName = Paths.get(item.getPagePath()).getFileName().toString();
                    outFileName = outFileName.replaceFirst("\\.html", "\\.pdf");
                    final File pdfFile = new File(outputDir.resolve(outFileName).toString());
                    final List<TOC.TOCItem> docToc = new ArrayList<>();
                    docToc.add(item);
                    try {
                        log.debug("reading " + item + "; generating pdf: " + pdfFile);
                    } catch (Exception e) {
                        messages.add(new DocetIssue(Severity.ERROR, "Impossible to generate pdf file. Reason: " + e));
                    }
                });
            } catch (IOException e) {
                messages.add(new DocetIssue(Severity.ERROR, "Impossible to read TOC. Reason: " + e));
            }
        } else {
            messages.add(new DocetIssue(Severity.WARN, "No folder found for language"));
        }
        result.put(lang, messages);
        return result;
    }

    private static TOC parseTOCFromPath(final Path tocFilePath, final Path tmpDir, final List<DocetIssue> messages) throws IOException {
        final TOC toc = new TOC();
        try (InputStream stream = Files.newInputStream(tocFilePath)) {
            final org.jsoup.nodes.Document htmlDoc = Jsoup.parseBodyFragment(FileUtils.readFileToString(tocFilePath.toFile(), ENCODING_UTF8));
            final Elements mainTOCItems = htmlDoc.select("nav#docet-menu>ul>li");
            mainTOCItems.forEach(el -> {
                final String fileName = el.getElementsByTag("a").get(0).attr("href");
                Path actualPagePath;
                try {
                    actualPagePath = searchFileInBasePathByName(tocFilePath.getParent().resolve(CONFIG_NAMES_FOLDER_PAGES), fileName);
                    final org.jsoup.nodes.Document sanitizedDoc = prepareHTMLForConversion(actualPagePath, tocFilePath.getParent());
                    actualPagePath = saveDocumentToDirectory(sanitizedDoc, actualPagePath.getFileName().toString(), tmpDir);
                    TOC.TOCItem item = new TOC.TOCItem(actualPagePath.toString());
                    populateTOCSubtree(tocFilePath, tmpDir, item, el.select("ul>li"), 1, messages);
                    toc.addItem(item);
                } catch (Exception e) {
                    messages.add(new DocetIssue(Severity.ERROR, "Error while parsing TOC. Reason: " + e));
                }
            });
        }
        return toc;
    }

    private static Path saveDocumentToDirectory(final org.jsoup.nodes.Document doc, final String fileName, final Path tmpDir) throws IOException {
        final Path outTmpPath = tmpDir.resolve(fileName);
        final File tmpFile = outTmpPath.toFile();
        FileUtils.writeStringToFile(tmpFile, doc.outerHtml(), ENCODING_UTF8);
        return outTmpPath;
    }

    private static void populateTOCSubtree(final Path tocFilePath, final Path tmpDir, final TOC.TOCItem item, final Elements domSubitems,
        final int level, final List<DocetIssue> messages) {
        if (domSubitems.isEmpty()) {
            return;
        }
        domSubitems.forEach(el -> {
            try {
                final String fileName = el.getElementsByTag("a").get(0).attr("href");
                Path actualPagePath = searchFileInBasePathByName(tocFilePath.getParent().resolve(CONFIG_NAMES_FOLDER_PAGES), fileName);
                final org.jsoup.nodes.Document sanitizedDoc = prepareHTMLForConversion(actualPagePath, tocFilePath.getParent());
                actualPagePath = saveDocumentToDirectory(sanitizedDoc, actualPagePath.getFileName().toString(), tmpDir);
                final TOC.TOCItem subItem = new TOC.TOCItem(actualPagePath.toString(), level);
                populateTOCSubtree(tocFilePath, tmpDir, subItem, el.select("ul>li"), level + 1, messages);
                item.addSubItem(subItem);
            } catch (Exception e) {
                messages.add(new DocetIssue(Severity.ERROR, "Error while parsing TOC. Reason: " + e));
            }
        });
    }

    private static org.jsoup.nodes.Document prepareHTMLForConversion(final Path htmlPagePath, final Path basePath) throws IOException {
        final org.jsoup.nodes.Document doc = Jsoup.parse(FileUtils.readFileToString(htmlPagePath.toFile(), ENCODING_UTF8), "", Parser.xmlParser());
        doc.getElementsByTag("head").append("<link href=\"src/docs/mndoc.css\" type=\"text/css\" rel=\"stylesheet\" />");
        final Elements images = doc.select("img");
        for (final Element img : images) {
            img.attr("src", searchFileInBasePathByName(basePath, img.attr("src")).toString());
        }
        return doc;
    }

    private static Path searchFileInBasePathByName(final Path basePath, final String fileName) throws IOException {
        final Holder<Path> result = new Holder<>(null);
        final String parsedFilename = fileName.split("#")[0];
        Files.walkFileTree(basePath, new SimpleFileVisitor<Path>() {
            @Override
            public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException {
                if (file.endsWith(parsedFilename)) {
                    result.setValue(file);
                    return FileVisitResult.TERMINATE;
                }
                return FileVisitResult.CONTINUE;
            }
        });
        return result.value;
    }

    public static int zippingDocs(final Path srcDir, final Path outDir, final Path indexDir, final boolean includeIndex, final Path zipFileName,
        final Map<Language, List<FaqEntry>> faqs, final Log log) throws MojoFailureException {
        final Holder<Integer> scannedDocs = new Holder<>(0);
        final FileToZipFilter filter = new FileToZipFilter();
        try (OutputStream fos = Files.newOutputStream(zipFileName, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING);
            ZipOutputStream zos = new ZipOutputStream(fos);) {
            Files.walkFileTree(srcDir, new SimpleFileVisitor<Path>() {
                @Override
                public FileVisitResult preVisitDirectory(final Path file, final BasicFileAttributes attrs) throws IOException {
                    if (file.toFile().getName().startsWith(".")) {
                        return FileVisitResult.SKIP_SUBTREE;
                    }
                    return FileVisitResult.CONTINUE;
                }

                @Override
                public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException {
                    final Path normalizedPath = srcDir.normalize().relativize(file.normalize());
                    if (file.getFileName().toString().startsWith(".")) {
                        return FileVisitResult.CONTINUE;
                    }
                    if (log.isDebugEnabled()) {
                        log.debug("Visiting file " + file.getFileName() + "; path " + normalizedPath);
                    }
                    if (filter.accept(file.toFile())) {
                        final Language lang = extractLanguageFromPath(file);
                        if (file.toFile().getName().equals(CONFIG_NAMES_FILE_TOC)) {
                            final Path tocPath = generateTocForFaq(outDir, file, lang, log);
                            writeFileToArchive(zos, srcDir.getParent().relativize(srcDir), tocPath);
                        } else {
                            writeFileToArchive(zos, srcDir.getParent().relativize(srcDir), file);
                        }
                    } else if (log.isDebugEnabled()) {
                        log.debug("Skipped " + file.getFileName());
                    }
                    scannedDocs.setValue(scannedDocs.getValue() + 1);
                    return FileVisitResult.CONTINUE;
                }
            });
            if (includeIndex) {
                Files.walkFileTree(indexDir, new SimpleFileVisitor<Path>() {
                    @Override
                    public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException {
                        if (file.getFileName().toString().startsWith(".")) {
                            return FileVisitResult.CONTINUE;
                        }
                        if (log.isDebugEnabled()) {
                            log.debug("Visiting index file " + file.getFileName() + "; path " + outDir.normalize().relativize(file.normalize()));
                        }
                        writeFileToArchive(zos, indexDir.getParent().relativize(indexDir), file);
                        scannedDocs.setValue(scannedDocs.getValue() + 1);
                        return FileVisitResult.CONTINUE;
                    }
                });
            }
        } catch (IOException e) {
            throw new MojoFailureException("Error while generating zip archive: " + e, e);
        }
        return scannedDocs.getValue();
    }

    private static Language extractLanguageFromPath(final Path path) {
        final String pathRegexSeparator;
        if ("\\".equals(File.separator)) {
            pathRegexSeparator = "\\\\";
        } else {
            pathRegexSeparator = File.separator;
        }
        final String languagePathPattern = "(it|en|fr)" + pathRegexSeparator + "(faq|imgs|pdf|pages|toc\\.html)";
        Pattern pattern = Pattern.compile(languagePathPattern);
        Matcher matcher = pattern.matcher(path.toString());
        String lang = "";
        while (matcher.find()) {
            lang = matcher.group(1);
        }
        return Language.getLanguageByCode(lang);
    }

    private static Path generateTocForFaq(final Path outDir, final Path tocFile, final Language lang, final Log log)
        throws IOException {
        final Path outFaqDir = outDir.resolve(lang.toString());
        Files.createDirectories(outFaqDir);
        final Path outTocFile = outFaqDir.resolve(CONFIG_NAMES_FILE_TOC);

        final org.jsoup.nodes.Document htmlDoc = Jsoup.parseBodyFragment(FileUtils.readFileToString(tocFile.toFile(), ENCODING_UTF8));
        if (log.isDebugEnabled()) {
            log.debug(htmlDoc.html());
        }
        final Element faqHomeLink = htmlDoc.getElementById(FAQ_HOME_ANCHOR_ID);
        if (faqHomeLink != null) {
            faqHomeLink.addClass("faq-a");
        }
        final Element faqTocAnchor = htmlDoc.getElementById(FAQ_TOC_ID);
        if (faqTocAnchor != null) {
            final Elements faqs = faqTocAnchor.select("a");
            faqs.stream().forEach(faq -> {
                faq.addClass("faq-a");
            });
        }
        try (Writer fw = new OutputStreamWriter(new FileOutputStream(outTocFile.toFile()), "UTF-8");
            BufferedWriter bw = new BufferedWriter(fw);) {
            bw.write(htmlDoc.html());
        }
        return outTocFile;

    }

    private static void writeFileToArchive(final ZipOutputStream zos, final Path baseSrcPath, final Path filePath) throws IOException {
        final byte[] buffer = new byte[1024];
        final String zipPath = baseSrcPath.resolve(extractLanguageRelativePath(filePath)).toString();
        final ZipEntry ze = new ZipEntry(zipPath);
        zos.putNextEntry(ze);
        try (FileInputStream in = new FileInputStream(filePath.toFile());) {
            int len;
            while ((len = in.read(buffer)) > 0) {
                zos.write(buffer, 0, len);
            }
        }
        zos.closeEntry();
    }

    private static String extractLanguageRelativePath(final Path absolutePath) {
        final String fileName;
        if (absolutePath != null && absolutePath.getFileName() != null) {
            fileName = absolutePath.getFileName().toString();
        } else {
            fileName = "";
        }
        final String pathRegexSeparator;
        if ("\\".equals(File.separator)) {
            pathRegexSeparator = "\\\\";
        } else {
            pathRegexSeparator = File.separator;
        }
        final String languagePathPattern = "(it|en|fr)" + pathRegexSeparator + "(faq|imgs|pages|pdf|toc\\.html)";
        Pattern pattern = Pattern.compile(languagePathPattern);
        final String pathString;
        if (absolutePath != null) {
            pathString = absolutePath.toString();
        } else {
            pathString = "";
        }
        Matcher matcher = pattern.matcher(pathString);
        String lang = "";
        String folderType = "";
        while (matcher.find()) {
            lang = matcher.group(1);
            folderType = matcher.group(2);
        }
        String result = lang + File.separator + folderType;
        if (result.endsWith(File.separator)) {
            result = fileName;
        } else if (!fileName.equals(CONFIG_NAMES_FILE_TOC)) {
            result += File.separator + fileName;
        }
        return result;
    }

    public static void indexDocs(final Path outDir, final Path srcDir, final Map<Language, List<FaqEntry>> faqs,
        final Log log, final boolean compact)
        throws MojoFailureException {
        for (Language lang : Language.values()) {
            Path langPath = srcDir.resolve(lang.toString());
            langPath = langPath.resolve(CONFIG_NAMES_FOLDER_PAGES);
            if (Files.exists(langPath) && Files.isDirectory(langPath)) {
                indexDocsForLanguage(outDir, langPath, lang, faqs.get(lang), log, compact);
            } else {
                log.warn("[" + lang + "] No folder found for language");
            }
        }
    }

    public static int indexDocsForLanguage(final Path outDir, final Path path, final Language lang,
        final List<FaqEntry> faqs, final Log log, final boolean compact)
        throws MojoFailureException {
        final Holder<Integer> indexedDocs = new Holder<>(0);
        try (Directory dir = FSDirectory.open(outDir);) {
            //build the analyzer specific for the given language
            //Stadard analyzer if language is not supported
            Analyzer analyzer = new AnalyzerBuilder().language(lang).build();
            IndexWriterConfig iwc = new IndexWriterConfig(analyzer);
            iwc.setOpenMode(OpenMode.CREATE_OR_APPEND);

            // Optional: for better indexing performance, if you
            // are indexing many documents, increase the RAM
            // buffer. But if you do this, increase the max heap
            // size to the JVM (eg add -Xmx512m or -Xmx1g):
            //
            // iwc.setRAMBufferSizeMB(256.0);
            try (IndexWriter writer = new IndexWriter(dir, iwc);) {
                log.info("[" + lang + "] Building search index for language");
                Files.walkFileTree(path, new SimpleFileVisitor<Path>() {
                    @Override
                    public FileVisitResult preVisitDirectory(final Path file, final BasicFileAttributes attrs) throws IOException {
                        if (file.toFile().getName().startsWith(".")) {
                            return FileVisitResult.SKIP_SUBTREE;
                        }
                        return FileVisitResult.CONTINUE;
                    }

                    @Override
                    public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException {
                        if (file.getFileName().toString().startsWith(".")) {
                            return FileVisitResult.CONTINUE;
                        }
                        log.debug("Visiting " + file);
                        try {
                            indexDoc(writer, file, attrs.lastModifiedTime().toMillis(), lang.toString(), log);
                            indexedDocs.setValue(indexedDocs.getValue() + 1);
                        } catch (Exception ex) {
                            log.warn("[" + lang + "] File " + file + " cannot be read.", ex);
                        }
                        return FileVisitResult.CONTINUE;
                    }
                });
                // indexing faqs
                Optional.ofNullable(faqs).orElseGet(() -> new ArrayList<>()).stream().forEach(faqPage -> {
                    try {
                        final Path faqFile = faqPage.getFaqPath();
                        indexFaqPage(writer, faqFile, faqPage.getTitle(), faqFile.toFile().lastModified(), lang.toString(), log);
                        indexedDocs.setValue(indexedDocs.getValue() + 1);
                    } catch (Exception ex) {
                        log.warn("[" + lang + "] FAQ File " + faqPage.getFaqPath() + " cannot be read.", ex);
                    }
                });

                if (compact) {
                    writer.forceMerge(1);
                }
                writer.commit();
                if (indexedDocs.getValue() == 0) {
                    log.warn("[" + lang + "] \t -> No docs found for language");
                }
            } catch (IOException e) {
                throw new MojoFailureException("Impossible to index Docet docs.", e);
            }
        } catch (IOException e) {
            throw new MojoFailureException("Impossible to index Docet docs.", e);
        }
        return indexedDocs.getValue();
    }

    /**
     *
     * @param file
     * @param lastModified
     * @param lang
     * @param docTitle
     * @param docAbstract
     * @param docType
     *
     * @throws IOException
     * @throws SAXException
     * @throws TikaException
     */
    private static void indexGenericDoc(final IndexWriter writer, final Path file, final long lastModified, final String lang, final String docTitle,
        final String docAbstract, final int docType, final Log log) throws IOException, SAXException, TikaException {
        Document doc = new Document();
        try (InputStream stream = Files.newInputStream(file)) {
            Field pathField = new StringField("path", file.toString(), Field.Store.YES);
            doc.add(pathField);
            doc.add(new LongField("modified", lastModified, Field.Store.NO));
            doc.add(new TextField("contents-" + lang, convertDocToText(stream), Field.Store.YES));
            doc.add(new StringField("language", lang, Field.Store.YES));
            doc.add(new StringField("id", constructPageIdFromFilePath(file), Field.Store.YES));
            doc.add(new StringField("title", docTitle, Field.Store.YES));
            doc.add(new IntField("doctype", docType, Field.Store.YES));
            doc.add(new TextField("abstract", docAbstract, Field.Store.YES));
        }

        if (writer.getConfig().getOpenMode() == OpenMode.CREATE) {
            log.info("[" + lang + "] Adding " + file + " to index");
            log.debug("document: " + doc);
            writer.addDocument(doc);
        } else {
            log.debug("[" + lang + "] Updating " + file + " to index");
            writer.updateDocument(new Term("path", file.toString()), doc);
        }
    }

    /**
     * Indexes a single document.
     *
     * @throws TikaException
     * @throws SAXException
     * @throws IOException
     *
     */
    private static void indexDoc(final IndexWriter writer, final Path file, final long lastModified, final String lang, final Log log)
        throws IOException, SAXException, TikaException {
        String docTitle = "";
        String excerpt = "...";
        try (InputStream stream = Files.newInputStream(file)) {
            final org.jsoup.nodes.Document htmlDoc = Jsoup.parseBodyFragment(FileUtils.readFileToString(file.toFile(), ENCODING_UTF8));
            docTitle = htmlDoc.getElementsByTag("h1").get(0).text();

            // conventionally take the first <p> and treat it as an abstract.
            final Elements pars = htmlDoc.select("div#main p#abstract");
            if (!pars.isEmpty()) {
                final Element abstractPar = pars.get(0);
                final String firstPar = abstractPar.text();

                final String abstractText;
                if (firstPar != null && !firstPar.isEmpty()) {
                    abstractText = firstPar;
                } else {
                    // was not possible to find an abstract
                    abstractText = "";
                }

                if (abstractText.length() <= SHORT_SEARCH_TEXT_DEFAULT_LENGTH) {
                    excerpt = abstractText;
                } else {
                    excerpt = abstractText.substring(0, SHORT_SEARCH_TEXT_DEFAULT_LENGTH) + "...";
                }
            }
        }
        indexGenericDoc(writer, file, lastModified, lang, docTitle, excerpt, INDEX_DOCTYPE_PAGE, log);
    }

    /**
     * Indexes a single faq page.
     *
     * @throws TikaException
     * @throws SAXException
     * @throws IOException
     *
     */
    private static void indexFaqPage(final IndexWriter writer, final Path file, final String faqTitle, final long lastModified, final String lang,
        final Log log) throws IOException, SAXException, TikaException {
        String excerpt = "";
        final StringBuilder excerptBuilder = new StringBuilder();
        try (InputStream stream = Files.newInputStream(file)) {
            final org.jsoup.nodes.Document htmlDoc = Jsoup.parseBodyFragment(FileUtils.readFileToString(file.toFile(), ENCODING_UTF8));

            // conventionally take the first <p> and treat it as an abstract.
            final Elements pars = htmlDoc.select("div.faq-item");
            for (Element faq : pars) {
                final Elements questions = faq.select(".question");
                final Elements answers = faq.select(".answer");
                String question = "";
                // TODO to remove once validation is implemented
                if (!questions.isEmpty()) {
                    question = "<b> " + questions.get(0).text() + "</b><br/>";
                }
                String answer = "";
                if (!answers.isEmpty()) {
                    String rawAnswer = answers.get(0).text();
                    if (rawAnswer.length() > SHORT_SEARCH_ANSWER_TEXT_DEFAULT_LENGTH) {
                        rawAnswer = rawAnswer.substring(0, SHORT_SEARCH_ANSWER_TEXT_DEFAULT_LENGTH) + "...";
                    }
                    answer = rawAnswer;
                }
                excerptBuilder.append(question).append(answer).append("<br/>");
                excerpt = excerptBuilder.toString();
                if (excerpt.length() >= SHORT_SEARCH_TEXT_DEFAULT_LENGTH) {
                    break;
                }
            }
        }
        indexGenericDoc(writer, file, lastModified, lang, "FAQ - " + faqTitle, excerpt, INDEX_DOCTYPE_FAQ, log);

    }

    private static String getFileSeparatorForString() {
        final String separatorStr;
        if ("\\".equals(File.separator)) {
            separatorStr = "\\\\";
        } else {
            separatorStr = "/";
        }
        return separatorStr;
    }

    private static String constructPageIdFromFilePath(final Path file) {
        final String[] tokens = file.toString().split(getFileSeparatorForString());
        final String fileName = tokens[tokens.length - 1];
        return fileName.split(".html")[0];
    }

    public static class Holder<T> {

        private T value;

        Holder(T value) {
            setValue(value);
        }

        T getValue() {
            return value;
        }

        void setValue(T value) {
            this.value = value;
        }
    }

    private static String convertDocToText(final InputStream streamDoc) throws IOException, SAXException, TikaException {
        AutoDetectParser parser = new AutoDetectParser();
        BodyContentHandler handler = new BodyContentHandler();
        Metadata metadata = new Metadata();
        parser.parse(streamDoc, handler, metadata);
        return handler.toString();
    }

    private static class FileToZipFilter implements FileFilter {

        private final String[] toSkipExtensions = new String[]{"css"};

        @Override
        public boolean accept(final File file) {
            for (final String extension : toSkipExtensions) {
                if (file.getName().toLowerCase().endsWith(extension)) {
                    return false;
                }
            }
            return true;
        }
    }

    private static class AnalyzerBuilder {

        private Language lang;

        public AnalyzerBuilder() {
            this.lang = Language.IT;
        }

        public AnalyzerBuilder language(final Language lang) {
            this.lang = lang;
            return this;
        }

        public Analyzer build() {
            final Analyzer analyzer;
            switch (this.lang) {
                case FR:
                    analyzer = new FrenchAnalyzer();
                    break;
                case IT:
                    analyzer = new ItalianAnalyzer();
                    break;
                case EN:
                default:
                    analyzer = new StandardAnalyzer();
            }
            return analyzer;
        }
    }
}


text:
Lee TV
Pool star Jeanette Lee's appearance on Fox Sports Net's "The Best Damn Sports Show Period," has been postponed. Lee's appearance on ESPN's "Unscripted with Chris Connelly" is still scheduled for April 4 or 5. Check your local listings for exact air times.
Also look for Lee (ESPN's third-sexiest athlete) in an upcoming issue of FHM magazine.
Fisher Breaks Corr's Winning Streak, and Her Own Losing Streak
Allison Fisher defeated first-seed Karen Corr, 7-1, to win the Women's Professional Billiard Association Cuetec Cues Players Championship. This is Fisher's first classic tour win since the WPBA Nationals in December 2000; her victory snapped Corr's winning streak of six consecutive WPBA Classic Tour titles.
The Cuetec Cues Players Championship was held March 15-17, at the Valley Forge Convention Center in King of Prussia, Pa. It was the first stop on the WPBA 2002 Classic Tour. Upcoming events on the tour include the Spring Classic on April 17-21 in Alpine, Calif. and the Midwest Classic on August 21-25 in East Peoria, Ill.
Fisher's winning check was $10,000, while Corr settled for $7,000 and second place, followed by Helena Thornfeldt with $5,000 for third.
WPBA Explains Dismissal
In a follow-up to a Feb. 8 press release from Trifecta Management, Women's Professional Billiard Association President Jan McWorter released an additional unsigned statement commenting on the dismissal of Board Member Dr. Diana Sorrentino. The statement cited "numerous major findings by the Board of impropriety in communications with WPBA sponsors and business partners, the pool industry, the billiards press, and numerous others with whom the WPBA interacts," and "Dr. Sorrentino's frequent violation of WPBA protocols for communication, and unwillingness to reform even after reprimand by the President of the organization," as the substantial reasons for Dr. Sorrentino's dismissal. The statement further stated, "the evidence of Dr. Sorrentino's impropriety is, almost entirely, hard documentary evidence."
Sorrentino had no comment on the dismissal, nor the controversy surrounding it.
In response to speculation within the billiards community that Sorrentino's dismissal was a result of statements made on the Billiards Digest Cue Chalk Board discussion forum, the statement distributed by McWorter emphasized that "the WPBA does not object, in principle, to Dr. Sorrentino's use of the Billiards Digest online forum," and "the WPBA recognizes that the online forum participants are among the most enthusiastic and most knowledgeable members of the pool community, and we understand and appreciate that they tend to give pool a good name."
The WPBA will not release any of the documentary evidence, according to the statement, out of respect for Sorrentino.
Sorrentino Voted Out
Feb. 8 press release from Trifecta Entertainment, the Women's Professional Billiard Association announced that the Board of Directors had unanimously voted to dismiss newly elected board member Dr. Diana Sorrentino. Sorrentino, a New York business consultant, was dismissed with cause, according to the release. WPBA officials declined further comment on the dismissal, pending formal correspondence with Dr. Sorrentino. In the weeks leading up to the ouster, WPBA board members scrutinized Internet discussion forums where Sorrentino was an active participant.
WPBA bylaws provide for removal of members "with or without cause at any time, by the vote of two-thirds of the members of the board of directors present and voting at that meeting called for that purpose." At press time it was not known how many board members voted on the issue.
"This has been an unpleasant process for the entire board but we are committed to furthering the goals and objectives of the WPBA," said Jan McWorter, WPBA president, in the press release. "We would like to thank Dr. Sorrentino for her service and wish her well in her future endeavors." McWorter added that the Board is in discussion about how to fill the open position, which they hope to do immediately. "In the past we've gone to the next-highest vote getter [in the most recent election]," said WPBA vice-president Kelly Oyama.
Scorpion Tops Dragon in D.C.
Johnny Archer registered a come-from-behind 9-8 victory in the 9-ball final of the Capital City Classic on Jan. 27 to win $6,500 for first place. The inaugural event marked the third straight major tournament of the 2002 season, following the Derby City Classic (Jan. 4-12) and Music City 9-Ball Open (Jan. 19-20).
Derby City one-pocket runner-up Jose Parica captured the one-pocket division at the Capital City Classic, topping Pete Fusco, 5-3, in the final to claim $6,000.
Charlie Williams led for most of the final against Johnny Archer, but Archer rallied to claim the victory. Williams settled for the $4,000 second-place check. Allen Hopkins finished third, worth $3,000.
Who Would You Pick, Jeanette Lee or Anna Kournikov
Jeanette Lee is used to facing tough seeded players, just usually not in the first round. But in ESPN's World's Sexiest Athletes poll, which is being conducted using a 32-athlete March Madness-style bracket, Lee has the misfortune of going head-to-head against one of the Internet's most-searched celebrities, Anna Kournikova. The Russian tennis star, who is reportedly married to Detroit Red Wings perennial National Hockey League All-Star Sergei Federov, has been consistently named Lycos' Most Searched Athlete, ahead of legends like Michael Jordan and Muhammad Ali.
Lee is the billiard community's most recognizable face, with numerous television and print appearances. Despite multiple surgeries last year, Lee has hardly missed a step, closing out the 2001 season with a runner-up finish behind No. 1-ranked Karen Corr at the Cuetec Cues National 9-Ball Championship.
To vote for Lee in the poll, log on to http://espn.go.com/sexiestathletes/. ESPN will televise the results during a two-hour special, The World's Sexiest Athletes presented by US Magazine, on Jan. 27 at 8 p.m. EST.
Team USA Captures Sixth Sraight Mosconi Cup
After Team Europe drew first blood in the 2001 Mosconi Cup, Team USA won the next 12 matches to claim its sixth straight Mosconi Cup title at York Hall in Bethnal Green, England, Dec. 20-23. The 12-1 victory for Team USA is the most decisive win in the eight-year history of the annual Ryder Cup-style event, which Team USA has now won seven times.
Only the pairing of Niels Feijen and Mika Immonen was able to scratch the scoreboard with a 5-3 triumph over veteran U.S. duo Johnny Archer and Nick Varner. But from then on, Team USA dominated play. The young American pair of Charlie Williams and Cory Deuel built Team USA's lead with two convincing victories over Steve Knight and Marcus Chamat by a combined score of 10-3. Archer and Varner then closed out the event by redeeming themselves from the Day 1 loss with a 5-1 win over Immonen and Team Europe captain Ralf Souquet.
Jimmy Caras placed in nursing home
Story by: Melinda Bailey – AzBilliards.com
Jimmy Caras, World Pool Champion, oldest living Hall-Of-Famer and #10 on Billiards Digest's 50 Greatest Players of the Century, had a stroke last week in Jacksonville, FL.
Jimmy was placed in a Nursing Home over the weekend and the first thing he asked the nurses was, Is there a pool table in here? Although Jimmy cannot walk or get out of bed now, he is still talking, joking and has all of his faculties.
Jimmy went to the poolroom in Jacksonville every afternoon that he could – even though his arthritis in his legs slowed him down. Jimmy turns 93 on December 17th.
Jimmy, who still loves pool but will now miss going to the poolroom everyday, would love to hear from fans and friends. Cards may be sent to his daughter's house at:
Jimmy Caras
3619 Bridgewood Dr.
Morris returns in Reno
After a long hiatus, Rodney Morris returned to his old form at the Sands Regency Reno Open, Dec. 4-9, topping Nick Varner in the final to claim $12,000.
Morris had been away from the game to serve a prison sentence for a narcotics-related conviction. Morris, who also competed in the Tokyo 9-Ball International Billiard Tournament the previous month, earned his last major victory by capturing the 1996 U.S. Open.
Varner collected $5,600 for second place, while Johnny Archer earned $3,700 for third.

redpajama_set_name: RedPajamaCommonCrawl


Perfect for coffee, wine, cocktails, tea, champagne, beer, ice cream and more, this 12 oz insulated wine tumbler with lid is ready to be offered as a mind-blowing gift!
GET A MODERN, EYE-CATCHING 12 Oz WINE TUMBLER that will add color to your life! Comfortable in the hand, it also fits in most custom cup holders, to follow you on your outdoor adventures! Worry no more about your little adventurers breaking their smoothie/juice/water or milkshake cups anymore! UNBREAKABLE and SAFE for kids, teenagers and adults, it is a cute, trendy and youthful colored metal wine glass, your #1 choice in home/camping/holiday/sports/ travel stainless wine glass to go!
Click here for more information about 'CHILLOUT LIFE Stainless Steel Stemless Wine Glass Tumbler with Lid, 12 oz | Double Wall Vacuum Insulated Travel Tumbler Cup for Coffee, Wine, Cocktails, Ice Cream | 2 Pack - Black'.


Stephen Few recently wrote a post after having attended an SAP BI presentation here in Melbourne a couple of weeks ago (unfortunately I missed it, and the chance to meet him, although a couple of colleagues from the DSS Lab did manage to catch up with him). The central thrust of his post centred on the issue of an engineering versus a design perspective on the problem of providing decision support.
I reckon he's spot on and his argument echoes a line of thought that I've been toying with for quite some time.
For myself, I've been grappling with this concept of design versus engineering as I've practised and taught IT-based design techniques like conceptual data modelling. What's fundamentally different between the two perspectives? Both engineering and design express themselves through the properties of a technical artefact. They also have a symbiotic relationship with each other - one can't do design without engineering, or vice versa. However, intuitively there is a fundamental difference of perspective, and it's taken me a couple of years of reading and thinking to put my finger on it.
The 'aha!' moment came a couple of weeks ago as I was working on a paper trying to more rigorously define the notion of Web 2.0. One of the problems with that idea is that every IT industry niche has jumped on the bandwagon so that now we have BI 2.0, ERP 2.0, even Library 2.0, without really identifying what it's fundamentally about.
My thesis is that Web 2.0 is not about the technology itself - AJAX, RoR etc. etc - because you can achieve the same Web 2.0-style outcomes using different technologies like HTML5 etc. Instead, Web 2.0 is a design concept, perhaps even a design movement. This led me, in a circuitous way, to read a paper by Lynne Markus and Mark Silver called A Foundation for the Study of IT Effects: A New Look at DeSanctis and Poole's Concepts of Structural Features and Spirit.1 In this paper, they critique an earlier theory of studying IT artefacts based on the idea that what's really important is not so much the features of a given piece of technology, but rather the relationship between that technology and its users.
For example, they draw on the concept of affordance. This relates to the things that an object or environment offers an actor, but it's specific to each actor. While a chair might be something that offers me the affordance of sitting, to a child it is an object that can be climbed on, crawled under, and so on. In other words, to understand a technological artefact, we have to look, not so much at the artefact itself, but how the properties of the artefact and the properties of the user interact.
And this was that aha! insight: the difference between design and engineering is that design is concerned with the space between the user and the artefact; engineers, on the other hand, are primarily concerned with the artefact itself. Both express themselves through the properties of the artefact itself, but design treats that as secondary to what the artefact does for the user, what it means, and how it impacts on the environment.
The problem with vendors in BI, and really most of the IT industry in general, is that we're all focussed on the technical artefact. As a result, the design of how that artefact is used, what it allows users to do, and the impact that it has on the decision support environment is largely ignored. As Stephen Few points out, old-school, large-scale BI vendors are so focussed on their software's bells and whistles that they almost completely ignore the fundamentals of helping people make more effective, better quality decisions.
Following from Markus and Silver's paper, I'd argue that design of IT artefacts, including BI software, needs to consider two key concepts: affordance, as described above, and what they call 'symbollic expressions.' The latter refers to the implicit values, principles and ideas communicated by the technology to the user. Too often in BI, the values and principles that are communicated by tools like Crystal Xcelsius, and lots of dashboard implementations, is form over substance. Dazzling your audience with 3D exploding pie charts and speed-dials is given more importance than imparting a proper understanding of the relevant information.
It's time that we had a design company in Business Intelligence, rather than the plethora of engineering companies we now have. Think Apple versus IBM. Love or hate them, there is no denying that Apple's focus is on the experience of using their products within an ecosystem of iTunes, App Store and iDevice, rather than on the technical attributes of the devices themselves. From a technical standpoint, Apple's devices are not significantly better (and they're often worse) than their competitor's products. But it's the design that goes into the experience of using the device that makes the fundamental difference. IBM, on the other hand, is the classic engineering company focussed primarily on the technical artefact itself.
We need a vendor to do for BI what Apple did for music players, mobile phones and tablet computing. We need a vendor that understands tactical and strategic decision making in organisations and who is willing to have a go at changing and improving that process rather than bolting on yet another suite of 'smoke and mirrors' to their already bloated software.
1 Markus, M. L. and M. S. Silver (2008) "A Foundation for the Study of IT Effects: A New Look at DeSanctis and Poole's Concepts of Structural Features and Spirit," Journal of the Association for Information Systems (9) 10/11, pp. 609-632.


Archives|Baseball Failed In Not Banning Drug; And the Pitchers?
Baseball Failed In Not Banning Drug; And the Pitchers?
It is worthwhile to try to learn whether androstenedione has helped Mark McGwire (Editorial Observer, Sept. 30) and whether there are long-term adverse effects. But when it comes to the question of whether this gave him an unfair advantage in hitting home runs, we also need to know the extent to which opposing pitchers were using it.


text:
Home World United Nations panel lauds India's digital initiatives for economic inclusion
United Nations panel lauds India's digital initiatives for economic inclusion
The United Nations panel, tasked to map the trends in digital technologies, identify gaps and opportunities and outline proposals for strengthening international cooperation
United Nations: India has been lauded by a high-level panel on digital cooperation launched by United Nations Secretary-General Antonio Guterres for undertaking revolutionary digital initiatives to ensure economic inclusion for its 1.3 billion citizens.
Launched in July last year by the UN Secretary-General, the 20-member panel, co-chaired by billionaire philanthropist Melinda Gates and Alibaba founder Jack Ma, in its report recognised the consequential role played by new digital ecosystem 'India Stack' in helping the government agencies and entrepreneurs achieve economic inclusion in India.
IndiaStack is a set of Application Programming Interface that allows governments, businesses, startups and developers to utilise a unique digital infrastructure to solve India's hard problems towards presence-less, paperless and cashless service delivery, its website said.
The panel, tasked to map the trends in digital technologies, identify gaps and opportunities and outline proposals for strengthening international cooperation.
It was launched to consider how digital cooperation can contribute to the achievement of the Sustainable Development Goals.
"Combinations of digital public goods can create 'common rails' for innovation of inclusive digital products and services," the panel's said in the report 'The Age of Digital Interdependence' at the UN headquarters this week.
"The India Stack is an example of how a unified, multi-layered software platform with clear standards, provided by public entities, can give government agencies and entrepreneurs the technological building blocks to improve service delivery and develop new business models which promote economic inclusion," it said.
The report by the Guterres-led United Nations body recognised the revolutionary digital initiatives taken by India to ensure economic inclusion for its 1.3 billion citizens.
The report stressed that digital cooperation must be tailor-made to meet the requirements of individual nations and communities, highlighting how India is achieving financial inclusion through its own digital platforms.
"What works in one country may not work in another. Rather than try to replicate specific successes, digital cooperation should aim to highlight best practices, standards and principles that can create conditions for local innovations to emerge and grow based on local issues, needs and cultural values," the report by the United Nations body said.
"India, for example, has added 300 million bank accounts in three years as new business models have been built on the India Stack, a set of government-managed online standards in areas including online payments and digital identity," the report said.
Introducing the report to the United Nations member States in the presence of the UN Chief, Minister of Digitalisation of Norway and a member of the panel Nikolai Astrup referred to India's unique identity infrastructure 'Aadhaar'.
"There can be great mechanisms for enhancing International digital cooperation. We call it digital public goods and the good news is that there are already great such technologies and content out there," Astrup said.
"The Modular Open Source Identity Platform (MOSIP) which has been developed by Indian stakeholders and a group of international donors based on learnings generated from India's implementation of its Aadhaar system and other early implementation of digital ID systems," Astrup said.
Speaking at an informal meeting of the General Assembly to hear a briefing by members of the panel, India's Permanent Representative to the United Nations Ambassador Syed Akbaruddin said the report highlighted that the spread of digital technologies has revolutionised the ability to communicate with others and to share and access knowledge.
It has also offered much-needed solutions for ending extreme poverty, to promoting inclusive economic growth and decent work to achieving universal literacy and doubling the productivity of small farmers, he said.
"While digital technologies are rapidly transforming societies and economies, they are also creating unprecedented challenges," Akbaruddin said.
"Digital technologies have brought new and serious concerns, including invasion of privacy, promotion of misinformation, infiltration of critical infrastructure through cyber-attacks, a threat to human rights and the issue of the growing digital divide," he said.
Akbaruddin noted that around the world there are some very polarising debates about data protection frameworks, human rights regimes and security policies.
"While the speed and scale of digitisation are increasing rapidly, the agility, responsiveness and scope of cooperation and governance mechanisms need some serious catching up to do," he said.
Akbaruddin said the unique benefits and profound risks arising from the dramatic increase in computing power and interconnectivity in the digital age fall in the realm of the "known unknowns".
"In many ways, your report seems to me is a call for enhanced multi-stakeholder digital cooperation, involving governments, civil society, academics, technologists, marginalised groups and the private sector. Such cooperation must be grounded in common human values such as inclusiveness, respect, human-centeredness, human rights, international law, transparency and sustainability," Akbaruddin said.
He emphasised that to capture the power of digital technologies, there is a need to cooperate on the broader ecosystems that enable digital technologies to be used in an inclusive manner.
"This will require policy frameworks that directly support economic and social inclusion," he said.
economic inclusion
Previous articleBishkek: Only multilateral, no bilateral, meetings for Modi
Next articleDMRC sends proposal for free rides
Big investors demand action on climate change ahead of G20 summit
Escalation in Gulf would be catastrophic: Guterres
Antonio Guterres: Hate speech spreading like wildfire via social media
Alibaba co-founder Jack Ma announces plans to retire at 54
UN Secretary General describes Vajpayee as an 'exemplary statesman'
Guterres taps Chile's 1st woman prez to be UN rights chief
UN chief calls new Muslim region in Philippines 'a landmark achievement'
India, USA start training of UN peacekeeping forces from Africa
Sirf News Network - 26 July 2016
India to grow at 7.2% in 2018, 7.4% in 2019: UN


In 2008, this Beaux Arts style theater was spared the wrecking ball by a non-profit company's $40,000 purchase with promises to renovate the building. They've since given tours and accepted donations to raise money, and we were given access as the last ever tour group to see the theater before renovations begin.
After years of decay, the theater was spared the wrecking ball by New York nonprofit company Western New York Minority Media Professionals, Inc., who plans to use the theater as a headquarters and production center for youth theater programs. To raise money for the restoration, they've given paid tours for photographers and visitors like us, and announced when we'd arrived that renovations were beginning right after our visit.
It was built to last on the site of a former, beloved theater that had burned down, and it was only its sturdy construction that preserved the structure enough to make renovation possible today. The lobby of the theater's purple paint is peeling back to reveal silver and gold swirled patterns, layered over years of change. The decorative molding over the doors has fallen away, showing a temporary, hand painted theatrical mask pattern, and the floor tiles beneath the dust are brightly colored and in good shape.
Through the doorway, past an assortment of seats, a sharp beam of light split through the oculus skylight in the ceiling, illuminating a spot in the audience. Keyboards and pianos had been left behind when the churches left, decaying with the structure, and one side of the balcony was too dangerous to reach. But the intricate details remained, with even some of the stained slag glass windows remaining unbroken in their sills.
The theater seats had been replaced over the years, featuring both ornate antique masthead motifs with intricately carved wreath patterns as well as sleek, vintage modern styles of the 30's and 40's.
Restoration work began that week. We look forward to going back to Buffalo for the opening gala, and to take some "after" shots!
The wood work on the chairs is beautiful, and I love the shots where the sunlight is just pouring in from the skylights!
Seating design has always peaked my interest when visiting a new theater. Thank you for including beautiful shots of this theater's detailed seats!
Would love to see a show there.
Stunning. You definitely captured the essence of modern ruins. There are tons of places: old buildings, farms, and more in upstate NY needing some documentary imagery 🙂 Thank you for sharing!
Thank you! I'm really looking forward to spending more time out there. Buffalo and J.N. Adams were our only stops in New York so far!
Just sublime, these photos are simply brilliant. The auditorium shots with light pouring in through the ( oculus? ) is spectacular, like a Piranesi etching!
Thank you, Andrew! It was indeed an oculus.


Airports Authority of India has replaced the existing Instrument Landing System (ILS) for runway 07 at Chennai International airport at a cost of Rs. 5.0 Crores. ILS provides the necessary guidance to pilots as they approach for landing at the runway. The new ILS will facilitate smooth landing of the aircrafts at Chennai Airport even during rain, low clouds, smog etc. and provide an improved guidance to the Aircrafts.
The new system uses a highly directional antenna system – "Wide Aperture Log Periodic Antenna Array [LPDA]" system. This antenna system has addressed the coverage restrictions of the old ILS, and will provide guidance to the aircraft to a much lower height and improved precision. This is turn will enhance the safety of the aircraft during landing.
The state of the art equipment is manufactured by M/s Indra Navia AS, Norway. The complete installation, alignment, testing, flight inspection and commissioning of the new Instrument Landing System has been done in-house by AAI's CNS Technical Experts.