<|start_filename|>src/utils/dom/mark-to-keep.js<|end_filename|>
import URL from 'url';

import { KEEP_SELECTORS, KEEP_CLASS } from './constants';

export default function markToKeep(article, $, url, tags = []) {
  if (tags.length === 0) {
    tags = KEEP_SELECTORS;
  }

  if (url) {
    const { protocol, hostname } = URL.parse(url);
    tags = [...tags, `iframe[src^="${protocol}//${hostname}"]`];
  }

  $(tags.join(','), article).addClass(KEEP_CLASS);

  return $;
}


<|start_filename|>src/extractors/generic/next-page-url/scoring/utils/score-prev-link.test.js<|end_filename|>
import assert from 'assert';

import scorePrevLink from './score-prev-link';

describe('scorePrevLink(linkData)', () => {
  it('returns -200 if link matches previous text', () => {
    const linkData = 'foo next previous page';

    assert.equal(scorePrevLink(linkData), -200);
  });

  it('returns 0 if does not match a prev link', () => {
    const linkData = 'foo bar WOW GREAT';

    assert.equal(scorePrevLink(linkData), 0);
  });
});


<|start_filename|>src/extractors/custom/www.refinery29.com/index.js<|end_filename|>
export const WwwRefinery29ComExtractor = {
  domain: 'www.refinery29.com',

  title: {
    selectors: ['h1.title'],
  },

  author: {
    selectors: ['.contributor'],
  },

  date_published: {
    selectors: [['meta[name="sailthru.date"]', 'value']],

    timezone: 'America/New_York',
  },

  lead_image_url: {
    selectors: [['meta[name="og:image"]', 'value']],
  },

  content: {
    selectors: [
      ['.full-width-opener', '.article-content'],
      '.article-content',
      '.body',
    ],

    // Is there anything in the content you selected that needs transformed
    // before it's consumable content? E.g., unusual lazy loaded images
    transforms: {
      'div.loading noscript': $node => {
        const imgHtml = $node.html();
        $node.parents('.loading').replaceWith(imgHtml);
      },

      '.section-image': 'figure',

      '.section-image .content-caption': 'figcaption',

      '.section-text': 'p',
    },

    // Is there anything that is in the result that shouldn't be?
    // The clean selectors will remove anything that matches from
    // the result
    clean: ['.story-share'],
  },
};


<|start_filename|>src/extractors/custom/www.si.com/index.js<|end_filename|>
export const WwwSiComExtractor = {
  domain: 'www.si.com',

  title: {
    selectors: ['h1', 'h1.headline'],
  },

  author: {
    selectors: [['meta[name="author"]', 'value']],
  },

  date_published: {
    selectors: ['.timestamp'],

    timezone: 'America/New_York',
  },

  dek: {
    selectors: ['.quick-hit ul'],
  },

  lead_image_url: {
    selectors: [['meta[name="og:image"]', 'value']],
  },

  content: {
    selectors: [['p', '.marquee_large_2x', '.component.image']],

    // Is there anything in the content you selected that needs transformed
    // before it's consumable content? E.g., unusual lazy loaded images
    transforms: {
      noscript: $node => {
        const $children = $node.children();
        if ($children.length === 1 && $children.get(0).tagName === 'img') {
          return 'figure';
        }

        return null;
      },
    },

    // Is there anything that is in the result that shouldn't be?
    // The clean selectors will remove anything that matches from
    // the result
    clean: [
      ['.inline-thumb', '.primary-message', '.description', '.instructions'],
    ],
  },
};


<|start_filename|>src/extractors/generic/content/scoring/find-top-candidate.js<|end_filename|>
import { NON_TOP_CANDIDATE_TAGS_RE } from './constants';
import { getScore } from './index';
import mergeSiblings from './merge-siblings';

// After we've calculated scores, loop through all of the possible
// candidate nodes we found and find the one with the highest score.
export default function findTopCandidate($) {
  let $candidate;
  let topScore = 0;

  $('[score]').each((index, node) => {
    // Ignore tags like BR, HR, etc
    if (NON_TOP_CANDIDATE_TAGS_RE.test(node.tagName)) {
      return;
    }

    const $node = $(node);
    const score = getScore($node);

    if (score > topScore) {
      topScore = score;
      $candidate = $node;
    }
  });

  // If we don't have a candidate, return the body
  // or whatever the first element is
  if (!$candidate) {
    return $('body') || $('*').first();
  }

  $candidate = mergeSiblings($candidate, topScore, $);

  return $candidate;
}


<|start_filename|>src/utils/dom/remove-empty.js<|end_filename|>
export default function removeEmpty($article, $) {
  $article.find('p').each((index, p) => {
    const $p = $(p);
    if ($p.find('iframe, img').length === 0 && $p.text().trim() === '')
      $p.remove();
  });

  return $;
}


<|start_filename|>src/extractors/generic/next-page-url/scoring/utils/score-cap-links.js<|end_filename|>
import { NEXT_LINK_TEXT_RE, CAP_LINK_TEXT_RE } from '../constants';

export default function scoreCapLinks(linkData) {
  // Cap links are links like "last", etc.
  if (CAP_LINK_TEXT_RE.test(linkData)) {
    // If we found a link like "last", but we've already seen that
    // this link is also "next", it's fine. If it's not been
    // previously marked as "next", then it's probably bad.
    // Penalize.
    if (NEXT_LINK_TEXT_RE.test(linkData)) {
      return -65;
    }
  }

  return 0;
}


<|start_filename|>rollup.config.web.js<|end_filename|>
import nodeResolve from 'rollup-plugin-node-resolve';
import globals from 'rollup-plugin-node-globals';
import { uglify } from 'rollup-plugin-uglify'; // eslint-disable-line import/extensions
import babel from 'rollup-plugin-babel';
import commonjs from 'rollup-plugin-commonjs';

export default {
  input: 'src/mercury.js',
  plugins: [
    babel({
      runtimeHelpers: true,
      exclude: './node_modules#<{(|*',
    }),
    commonjs({
      ignoreGlobal: true,
    }),
    globals(),
    nodeResolve({
      browser: true,
      preferBuiltins: false,
    }),
    uglify(),
  ],
  treeshake: true,
  output: {
    file: process.env.MERCURY_TEST_BUILD
      ? 'dist/mercury_test.web.js'
      : 'dist/mercury.web.js',
    format: 'iife',
    name: 'Mercury',
    sourceMap: true,
  },
};


<|start_filename|>src/extractors/generic/next-page-url/scoring/utils/should-score.js<|end_filename|>
import URL from 'url';

import { DIGIT_RE, EXTRANEOUS_LINK_HINTS_RE } from '../constants';

export default function shouldScore(
  href,
  articleUrl,
  baseUrl,
  parsedUrl,
  linkText,
  previousUrls
) {
  // skip if we've already fetched this url
  if (previousUrls.find(url => href === url) !== undefined) {
    return false;
  }

  // If we've already parsed this URL, or the URL matches the base
  // URL, or is empty, skip it.
  if (!href || href === articleUrl || href === baseUrl) {
    return false;
  }

  const { hostname } = parsedUrl;
  const { hostname: linkHost } = URL.parse(href);

  // Domain mismatch.
  if (linkHost !== hostname) {
    return false;
  }

  // If href doesn't contain a digit after removing the base URL,
  // it's certainly not the next page.
  const fragment = href.replace(baseUrl, '');
  if (!DIGIT_RE.test(fragment)) {
    return false;
  }

  // This link has extraneous content (like "comment") in its link
  // text, so we skip it.
  if (EXTRANEOUS_LINK_HINTS_RE.test(linkText)) {
    return false;
  }

  // Next page link text is never long, skip if it is too long.
  if (linkText.length > 25) {
    return false;
  }

  return true;
}


<|start_filename|>src/extractors/generic/next-page-url/scoring/utils/score-similarity.test.js<|end_filename|>
import assert from 'assert';

import scoreSimilarity from './score-similarity';

describe('scoreSimilarity(score, articleUrl, href)', () => {
  it('returns a similarity bonus based on current score', () => {
    const articleUrl = 'http://example.com/foo/bar';
    const href = 'http://example.com/foo/bar/2';
    const score = 25;
    assert.equal(Math.round(scoreSimilarity(score, articleUrl, href)), 66);
  });

  it('returns 0 is current score <= 0', () => {
    const articleUrl = 'http://example.com/foo/bar';
    const href = 'http://example.com/foo/bar/2';
    const score = 0;
    assert.equal(scoreSimilarity(score, articleUrl, href), 0);
  });
});


<|start_filename|>src/resource/utils/index.js<|end_filename|>
export { default as fetchResource } from './fetch-resource';


<|start_filename|>src/cleaners/dek.js<|end_filename|>
import { stripTags } from 'utils/dom';
import { excerptContent, normalizeSpaces } from 'utils/text';

import { TEXT_LINK_RE } from './constants';

// Take a dek HTML fragment, and return the cleaned version of it.
// Return None if the dek wasn't good enough.
export default function cleanDek(dek, { $, excerpt }) {
  // Sanity check that we didn't get too short or long of a dek.
  if (dek.length > 1000 || dek.length < 5) return null;

  // Check that dek isn't the same as excerpt
  if (excerpt && excerptContent(excerpt, 10) === excerptContent(dek, 10))
    return null;

  const dekText = stripTags(dek, $);

  // Plain text links shouldn't exist in the dek. If we have some, it's
  // not a good dek - bail.
  if (TEXT_LINK_RE.test(dekText)) return null;

  return normalizeSpaces(dekText.trim());
}


<|start_filename|>src/utils/text/page-num-from-url.js<|end_filename|>
import { PAGE_IN_HREF_RE } from './constants';

export default function pageNumFromUrl(url) {
  const matches = url.match(PAGE_IN_HREF_RE);
  if (!matches) return null;

  const pageNum = parseInt(matches[6], 10);

  // Return pageNum < 100, otherwise
  // return null
  return pageNum < 100 ? pageNum : null;
}


<|start_filename|>src/utils/text/remove-anchor.test.js<|end_filename|>
import assert from 'assert';

import removeAnchor from './remove-anchor';

describe('removeAnchor(url)', () => {
  it('returns a url w/out #anchor', () => {
    const url = 'http://example.com/foo/bar/wow-cool/page=10/#wow';
    const cleaned = 'http://example.com/foo/bar/wow-cool/page=10';

    assert.equal(removeAnchor(url), cleaned);
  });

  it('returns same url if url has no anchor found', () => {
    const url = 'http://example.com/foo/bar/wow-cool';
    const cleaned = 'http://example.com/foo/bar/wow-cool';

    assert.equal(removeAnchor(url), cleaned);
  });
});


<|start_filename|>src/extractors/generic/lead-image-url/extractor.test.js<|end_filename|>
import assert from 'assert';
import cheerio from 'cheerio';

import HTML from './fixtures/html';

import GenericLeadImageUrlExtractor from './extractor';

describe('GenericLeadImageUrlExtractor', () => {
  describe('extract({ $, content, metaCache })', () => {
    it('returns og:image first', () => {
      const $ = cheerio.load(HTML.og.test);
      const content = $('*').first();
      const metaCache = ['og:image'];

      const result = GenericLeadImageUrlExtractor.extract({
        $,
        content,
        metaCache,
      });

      assert.equal(result, HTML.og.result);
    });

    it('returns twitter:image', () => {
      const $ = cheerio.load(HTML.twitter.test);
      const content = $('*').first();
      const metaCache = ['twitter:image'];

      const result = GenericLeadImageUrlExtractor.extract({
        $,
        content,
        metaCache,
      });

      assert.equal(result, HTML.twitter.result);
    });

    it('finds images based on scoring', () => {
      const $ = cheerio.load(HTML.scoring.test);
      const content = $('*').first();
      const metaCache = [];

      const result = GenericLeadImageUrlExtractor.extract({
        $,
        content,
        metaCache,
      });

      assert.equal(result, HTML.scoring.result);
    });

    it('returns image based on selectors', () => {
      const $ = cheerio.load(HTML.selectors.test);
      const content = $('*').first();
      const metaCache = [];

      const result = GenericLeadImageUrlExtractor.extract({
        $,
        content,
        metaCache,
      });

      assert.equal(result, HTML.selectors.result);
    });
  });
});


<|start_filename|>src/utils/dom/strip-unlikely-candidates.js<|end_filename|>
import { CANDIDATES_WHITELIST, CANDIDATES_BLACKLIST } from './constants';

export default function stripUnlikelyCandidates($) {
  //  Loop through the provided document and remove any non-link nodes
  //  that are unlikely candidates for article content.
  //
  //  Links are ignored because there are very often links to content
  //  that are identified as non-body-content, but may be inside
  //  article-like content.
  //
  //  :param $: a cheerio object to strip nodes from
  //  :return $: the cleaned cheerio object
  $('*')
    .not('a')
    .each((index, node) => {
      const $node = $(node);
      const classes = $node.attr('class');
      const id = $node.attr('id');
      if (!id && !classes) return;

      const classAndId = `${classes || ''} ${id || ''}`;
      if (CANDIDATES_WHITELIST.test(classAndId)) {
        return;
      }
      if (CANDIDATES_BLACKLIST.test(classAndId)) {
        $node.remove();
      }
    });

  return $;
}


<|start_filename|>src/extractors/custom/uproxx.com/index.js<|end_filename|>
export const UproxxComExtractor = {
  domain: 'uproxx.com',

  title: {
    selectors: ['div.post-top h1'],
  },

  author: {
    selectors: ['.post-top .authorname'],
  },

  date_published: {
    selectors: [['meta[name="article:published_time"]', 'value']],
  },

  lead_image_url: {
    selectors: [['meta[name="og:image"]', 'value']],
  },

  content: {
    selectors: ['.post-body'],

    // Is there anything in the content you selected that needs transformed
    // before it's consumable content? E.g., unusual lazy loaded images
    transforms: {
      'div.image': 'figure',
      'div.image .wp-media-credit': 'figcaption',
    },

    // Is there anything that is in the result that shouldn't be?
    // The clean selectors will remove anything that matches from
    // the result
    clean: [],
  },
};


<|start_filename|>src/extractors/generic/excerpt/constants.js<|end_filename|>
export const EXCERPT_META_SELECTORS = ['og:description', 'twitter:description'];


<|start_filename|>src/utils/dom/extract-from-meta.test.js<|end_filename|>
import assert from 'assert';
import cheerio from 'cheerio';

import HTML from './fixtures/extract-from-selectors';
import { extractFromMeta } from './index';

describe('extractFromMeta($, metaNames, cachedNames, cleanTags)', () => {
  it('extracts an arbitrary meta tag by name', () => {
    const $ = cheerio.load(HTML.metaFoo.test);
    const result = extractFromMeta($, ['foo', 'baz'], ['foo', 'bat']);

    assert.equal(result, HTML.metaFoo.result);
  });

  it('returns nothing if a meta name is duplicated', () => {
    const $ = cheerio.load(HTML.metaDupes.test);
    const result = extractFromMeta($, ['foo', 'baz'], ['foo', 'bat']);

    assert.equal(result, HTML.metaDupes.result);
  });

  it('ignores duplicate meta names with empty values', () => {
    const $ = cheerio.load(HTML.metaEmptyDupes.test);
    const result = extractFromMeta($, ['foo', 'baz'], ['foo', 'bat']);

    assert.equal(result, HTML.metaEmptyDupes.result);
  });
});


<|start_filename|>src/extractors/generic/url/extractor.js<|end_filename|>
import URL from 'url';
import { extractFromMeta } from 'utils/dom';

import { CANONICAL_META_SELECTORS } from './constants';

function parseDomain(url) {
  const parsedUrl = URL.parse(url);
  const { hostname } = parsedUrl;
  return hostname;
}

function result(url) {
  return {
    url,
    domain: parseDomain(url),
  };
}

const GenericUrlExtractor = {
  extract({ $, url, metaCache }) {
    const $canonical = $('link[rel=canonical]');
    if ($canonical.length !== 0) {
      const href = $canonical.attr('href');
      if (href) {
        return result(href);
      }
    }

    const metaUrl = extractFromMeta($, CANONICAL_META_SELECTORS, metaCache);
    if (metaUrl) {
      return result(metaUrl);
    }

    return result(url);
  },
};

export default GenericUrlExtractor;


<|start_filename|>src/cleaners/lead-image-url.js<|end_filename|>
import validUrl from 'valid-url';

export default function clean(leadImageUrl) {
  leadImageUrl = leadImageUrl.trim();
  if (validUrl.isWebUri(leadImageUrl)) {
    return leadImageUrl;
  }

  return null;
}


<|start_filename|>src/resource/utils/dom/index.js<|end_filename|>
export { default as normalizeMetaTags } from './normalize-meta-tags';
export {
  default as convertLazyLoadedImages,
} from './convert-lazy-loaded-images';
export { default as clean } from './clean';


<|start_filename|>src/extractors/generic/next-page-url/scoring/utils/score-link-text.test.js<|end_filename|>
import assert from 'assert';

import scoreLinkText from './score-link-text';

describe('scoreLinkText(linkText)', () => {
  it('returns 8 if link contains the num 2', () => {
    assert.equal(scoreLinkText('2', 0), 8);
  });

  it('returns 5 if link contains the num 5', () => {
    assert.equal(scoreLinkText('5', 0), 5);
  });

  it('returns -30 if link contains the number 1', () => {
    assert.equal(scoreLinkText('1', 0), -30);
  });

  it('penalizes -50 if pageNum is >= link text as num', () => {
    assert.equal(scoreLinkText('4', 5), -44);
  });
});


<|start_filename|>src/extractors/custom/www.apartmenttherapy.com/index.js<|end_filename|>
// Rename CustomExtractor
// to fit your publication
// (e.g., NYTimesExtractor)
export const ApartmentTherapyExtractor = {
  domain: 'www.apartmenttherapy.com',
  title: {
    selectors: ['h1.headline'],
  },

  author: {
    selectors: ['.PostByline__name'],
  },

  content: {
    selectors: ['div.post__content'],

    // Is there anything in the content you selected that needs transformed
    // before it's consumable content? E.g., unusual lazy loaded images
    transforms: {
      'div[data-render-react-id="images/LazyPicture"]': ($node, $) => {
        const data = JSON.parse($node.attr('data-props'));
        const { src } = data.sources[0];
        const $img = $('<img />').attr('src', src);
        $node.replaceWith($img);
      },
    },

    // Is there anything that is in the result that shouldn't be?
    // The clean selectors will remove anything that matches from
    // the result
    clean: [],
  },

  date_published: {
    selectors: [['.PostByline__timestamp[datetime]', 'datetime']],
  },

  lead_image_url: {
    selectors: [['meta[name="og:image"]', 'value']],
  },

  dek: {
    selectors: [],
  },

  next_page_url: {
    selectors: [
      // enter selectors
    ],
  },

  excerpt: {
    selectors: [
      // enter selectors
    ],
  },
};


<|start_filename|>src/extractors/custom/www.fortinet.com/index.js<|end_filename|>
export const WwwFortinetComExtractor = {
  domain: 'www.fortinet.com',

  title: {
    selectors: ['h1'],
  },

  author: {
    selectors: ['.b15-blog-meta__author'],
  },

  date_published: {
    selectors: [['meta[name="article:published_time"]', 'value']],
  },

  lead_image_url: {
    selectors: [['meta[name="og:image"]', 'value']],
  },

  content: {
    selectors: [
      'div.responsivegrid.aem-GridColumn.aem-GridColumn--default--12',
    ],

    transforms: {
      noscript: $node => {
        const $children = $node.children();
        if ($children.length === 1 && $children.get(0).tagName === 'img') {
          return 'figure';
        }
        return null;
      },
    },
  },
};


<|start_filename|>src/utils/dom/set-attr.test.js<|end_filename|>
import assert from 'assert';

import { MockDomNode } from 'test-helpers';
import setAttr from './set-attr';

describe('setAttr(node, attr, val)', () => {
  it('sets attrs for a raw jquery node', () => {
    const domNode = new MockDomNode();

    const node = setAttr(domNode, 'class', 'foo');

    assert.equal(node.attributes[0].value, 'foo');
  });

  it('sets attrs for a raw cheerio node', () => {
    const cheerioNode = {
      attribs: {
        class: 'foo bar',
        id: 'baz bat',
      },
    };

    const node = setAttr(cheerioNode, 'class', 'foo');

    assert.equal(node.attribs.class, 'foo');
  });
});


<|start_filename|>src/extractors/generic/author/fixtures/html.js<|end_filename|>
const HTML = {
  authorMeta: {
    test: `
      <html>
        <meta name="dc.author" value="Adam" />
      </html>
    `,
    result: 'Adam',
  },
  authorSelectors: {
    test: `
      <div>
        <div class="byline">
          <a href="/author/adam">Adam</a>
        </div>
      </div>
    `,
    result: 'Adam',
  },
  authorRegSelectors: {
    test: `
      <div>
        <div class="byline">
          <span>By Adam</span>
        </div>
      </div>
    `,
    result: 'Adam',
  },
};

export default HTML;


<|start_filename|>src/extractors/custom/www.msnbc.com/index.js<|end_filename|>
export const WwwMsnbcComExtractor = {
  domain: 'www.msnbc.com',

  title: {
    selectors: ['h1', 'h1.is-title-pane'],
  },

  author: {
    selectors: ['.author'],
  },

  date_published: {
    selectors: [['meta[name="DC.date.issued"]', 'value']],
  },

  dek: {
    selectors: [['meta[name="description"]', 'value']],
  },

  lead_image_url: {
    selectors: [['meta[name="og:image"]', 'value']],
  },

  content: {
    selectors: ['.pane-node-body'],

    // Is there anything in the content you selected that needs transformed
    // before it's consumable content? E.g., unusual lazy loaded images
    transforms: {
      '.pane-node-body': ($node, $) => {
        const [
          selector,
          attr,
        ] = WwwMsnbcComExtractor.lead_image_url.selectors[0];
        const src = $(selector).attr(attr);
        if (src) {
          $node.prepend(`<img src="${src}" />`);
        }
      },
    },

    // Is there anything that is in the result that shouldn't be?
    // The clean selectors will remove anything that matches from
    // the result
    clean: [],
  },
};


<|start_filename|>src/extractors/generic/next-page-url/scoring/utils/score-next-link-text.test.js<|end_filename|>
import assert from 'assert';

import scoreNextLinkText from './score-next-link-text';

describe('scoreNextLinkText(linkData)', () => {
  it('returns 50 if contains common next link text', () => {
    const linkData = 'foo bar Next page';

    assert.equal(scoreNextLinkText(linkData), 50);
  });

  it('returns 0 if does not contain common next link text', () => {
    const linkData = 'foo bar WOW GREAT';

    assert.equal(scoreNextLinkText(linkData), 0);
  });
});


<|start_filename|>src/utils/dom/rewrite-top-level.test.js<|end_filename|>
import cheerio from 'cheerio';
import assert from 'assert';

import { assertClean } from 'test-helpers';

import HTML from './fixtures/html';
import rewriteTopLevel from './rewrite-top-level';

describe('rewriteTopLevel(node, $)', () => {
  it('turns html and body tags into divs', () => {
    const $ = cheerio.load(HTML.rewriteHTMLBody.before);
    const result = rewriteTopLevel($('html').first(), $);

    assert.equal(result('html').length, 0);
    assert.equal(result('body').length, 0);

    if (!cheerio.browser) {
      assertClean(result.html(), HTML.rewriteHTMLBody.after);
    }
  });
});


<|start_filename|>src/extractors/custom/www.cnet.com/index.js<|end_filename|>
export const WwwCnetComExtractor = {
  domain: 'www.cnet.com',

  title: {
    selectors: [['meta[name="og:title"]', 'value']],
  },

  author: {
    selectors: ['a.author'],
  },

  date_published: {
    selectors: ['time'],

    timezone: 'America/Los_Angeles',
  },

  dek: {
    selectors: ['.article-dek'],
  },

  lead_image_url: {
    selectors: [['meta[name="og:image"]', 'value']],
  },

  content: {
    selectors: [
      ['img.__image-lead__', '.article-main-body'],
      '.article-main-body',
    ],

    // Is there anything in the content you selected that needs transformed
    // before it's consumable content? E.g., unusual lazy loaded images
    transforms: {
      'figure.image': $node => {
        const $img = $node.find('img');
        $img.attr('width', '100%');
        $img.attr('height', '100%');
        $img.addClass('__image-lead__');
        $node.remove('.imgContainer').prepend($img);
      },
    },

    // Is there anything that is in the result that shouldn't be?
    // The clean selectors will remove anything that matches from
    // the result
    clean: [],
  },
};


<|start_filename|>src/utils/dom/strip-unlikely-candidates.test.js<|end_filename|>
import assert from 'assert';
import cheerio from 'cheerio';

import { assertClean } from 'test-helpers';
import HTML from './fixtures/html';
import stripUnlikelyCandidates from './strip-unlikely-candidates';

function assertBeforeAndAfter(key, fn) {
  const $ = cheerio.load(HTML[key].before);
  assertClean(fn($).html(), HTML[key].after);
}

describe('Generic Extractor Utils', () => {
  describe('stripUnlikelyCandidates(node)', () => {
    it('returns original doc if no matches found', () => {
      const $ = cheerio.load(HTML.noMatches);
      const stripped = stripUnlikelyCandidates($);
      assert.equal(stripped.html(), HTML.noMatches);
    });

    it('strips unlikely matches from the doc', () => {
      assertBeforeAndAfter('whitelistMatch', stripUnlikelyCandidates);
    });

    it('keeps likely matches even when they also match the blacklist', () => {
      assertBeforeAndAfter('whiteAndBlack', stripUnlikelyCandidates);
    });

    it('removed likely matches when inside blacklist node', () => {
      assertBeforeAndAfter('whiteInsideBlack', stripUnlikelyCandidates);
    });
  });
});


<|start_filename|>scripts/templates/insert-values.js<|end_filename|>
export default function insertValues(strings, ...values) {
  if (values.length) {
    return strings.reduce((result, part, idx) => {
      let value = values[idx];

      if (value && typeof value.toString === 'function') {
        value = value.toString();
      } else {
        value = '';
      }

      return result + part + value;
    }, '');
  }

  return strings.join('');
}


<|start_filename|>src/extractors/custom/www.broadwayworld.com/index.js<|end_filename|>
// Rename CustomExtractor
// to fit your publication
// (e.g., NYTimesExtractor)
export const BroadwayWorldExtractor = {
  domain: 'www.broadwayworld.com',
  title: {
    selectors: ['h1.article-title'],
  },

  author: {
    selectors: ['span[itemprop=author]'],
  },

  content: {
    selectors: ['div[itemprop=articlebody]'],

    // Is there anything in the content you selected that needs transformed
    // before it's consumable content? E.g., unusual lazy loaded images
    transforms: {},

    // Is there anything that is in the result that shouldn't be?
    // The clean selectors will remove anything that matches from
    // the result
    clean: [],
  },

  date_published: {
    selectors: [['meta[itemprop=datePublished]', 'value']],
  },

  lead_image_url: {
    selectors: [['meta[name="og:image"]', 'value']],
  },

  dek: {
    selectors: [],
  },

  next_page_url: {
    selectors: [
      // enter selectors
    ],
  },

  excerpt: {
    selectors: [
      // enter selectors
    ],
  },
};


<|start_filename|>src/utils/dom/brs-to-ps.js<|end_filename|>
import { paragraphize } from './index';

// ## NOTES:
// Another good candidate for refactoring/optimizing.
// Very imperative code, I don't love it. - AP

//  Given cheerio object, convert consecutive <br /> tags into
//  <p /> tags instead.
//
//  :param $: A cheerio object

export default function brsToPs($) {
  let collapsing = false;
  $('br').each((index, element) => {
    const $element = $(element);
    const nextElement = $element.next().get(0);

    if (nextElement && nextElement.tagName.toLowerCase() === 'br') {
      collapsing = true;
      $element.remove();
    } else if (collapsing) {
      collapsing = false;
      paragraphize(element, $, true);
    }
  });

  return $;
}


<|start_filename|>src/extractors/generic/next-page-url/scoring/score-links.test.js<|end_filename|>
import assert from 'assert';
import cheerio from 'cheerio';

import scoreLinks from './score-links';

const fs = require('fs');

describe('scoreLinks(links)', () => {
  it('returns an object of scored links', () => {
    const html = fs.readFileSync('./fixtures/ars.html', 'utf8');

    const $ = cheerio.load(html);
    const links = $('a[href]').toArray();
    const url =
      'http://arstechnica.com/gadgets/2016/08/the-connected-renter-how-to-make-your-apartment-smarter/';

    const scoredPages = scoreLinks({
      links,
      articleUrl: url,
      baseUrl: 'http://arstechnica.com',
      $,
    });

    assert.equal(typeof scoredPages, 'object');
  });

  it('returns null if no possible pages', () => {
    const html = '<div><p>Hello wow</p></div>';

    const $ = cheerio.load(html);
    const links = $('a[href]').toArray();
    const url =
      'http://arstechnica.com/gadgets/2016/08/the-connected-renter-how-to-make-your-apartment-smarter/';

    const scoredPages = scoreLinks({
      links,
      articleUrl: url,
      baseUrl: 'http://arstechnica.com',
      $,
    });

    assert.equal(scoredPages, null);
  });
});


<|start_filename|>src/extractors/generic/content/scoring/constants.js<|end_filename|>
// // CONTENT FETCHING CONSTANTS ////

// A list of strings that can be considered unlikely candidates when
// extracting content from a resource. These strings are joined together
// and then tested for existence using re:test, so may contain simple,
// non-pipe style regular expression queries if necessary.
export const UNLIKELY_CANDIDATES_BLACKLIST = [
  'ad-break',
  'adbox',
  'advert',
  'addthis',
  'agegate',
  'aux',
  'blogger-labels',
  'combx',
  'comment',
  'conversation',
  'disqus',
  'entry-unrelated',
  'extra',
  'foot',
  'form',
  'header',
  'hidden',
  'loader',
  'login', // Note: This can hit 'blogindex'.
  'menu',
  'meta',
  'nav',
  'pager',
  'pagination',
  'predicta', // readwriteweb inline ad box
  'presence_control_external', // lifehacker.com container full of false positives
  'popup',
  'printfriendly',
  'related',
  'remove',
  'remark',
  'rss',
  'share',
  'shoutbox',
  'sidebar',
  'sociable',
  'sponsor',
  'tools',
];

// A list of strings that can be considered LIKELY candidates when
// extracting content from a resource. Essentially, the inverse of the
// blacklist above - if something matches both blacklist and whitelist,
// it is kept. This is useful, for example, if something has a className
// of "rss-content entry-content". It matched 'rss', so it would normally
// be removed, however, it's also the entry content, so it should be left
// alone.
//
// These strings are joined together and then tested for existence using
// re:test, so may contain simple, non-pipe style regular expression queries
// if necessary.
export const UNLIKELY_CANDIDATES_WHITELIST = [
  'and',
  'article',
  'body',
  'blogindex',
  'column',
  'content',
  'entry-content-asset',
  'format', // misuse of form
  'hfeed',
  'hentry',
  'hatom',
  'main',
  'page',
  'posts',
  'shadow',
];

// A list of tags which, if found inside, should cause a <div /> to NOT
// be turned into a paragraph tag. Shallow div tags without these elements
// should be turned into <p /> tags.
export const DIV_TO_P_BLOCK_TAGS = [
  'a',
  'blockquote',
  'dl',
  'div',
  'img',
  'p',
  'pre',
  'table',
].join(',');

// A list of tags that should be ignored when trying to find the top candidate
// for a document.
export const NON_TOP_CANDIDATE_TAGS = [
  'br',
  'b',
  'i',
  'label',
  'hr',
  'area',
  'base',
  'basefont',
  'input',
  'img',
  'link',
  'meta',
];

export const NON_TOP_CANDIDATE_TAGS_RE = new RegExp(
  `^(${NON_TOP_CANDIDATE_TAGS.join('|')})$`,
  'i'
);

// A list of selectors that specify, very clearly, either hNews or other
// very content-specific style content, like Blogger templates.
// More examples here: http://microformats.org/wiki/blog-post-formats
export const HNEWS_CONTENT_SELECTORS = [
  ['.hentry', '.entry-content'],
  ['entry', '.entry-content'],
  ['.entry', '.entry_content'],
  ['.post', '.postbody'],
  ['.post', '.post_body'],
  ['.post', '.post-body'],
];

export const PHOTO_HINTS = ['figure', 'photo', 'image', 'caption'];
export const PHOTO_HINTS_RE = new RegExp(PHOTO_HINTS.join('|'), 'i');

// A list of strings that denote a positive scoring for this content as being
// an article container. Checked against className and id.
//
// TODO: Perhaps have these scale based on their odds of being quality?
export const POSITIVE_SCORE_HINTS = [
  'article',
  'articlecontent',
  'instapaper_body',
  'blog',
  'body',
  'content',
  'entry-content-asset',
  'entry',
  'hentry',
  'main',
  'Normal',
  'page',
  'pagination',
  'permalink',
  'post',
  'story',
  'text',
  '[-_]copy', // usatoday
  '\\Bcopy',
];

// The above list, joined into a matching regular expression
export const POSITIVE_SCORE_RE = new RegExp(
  POSITIVE_SCORE_HINTS.join('|'),
  'i'
);

// Readability publisher-specific guidelines
export const READABILITY_ASSET = new RegExp('entry-content-asset', 'i');

// A list of strings that denote a negative scoring for this content as being
// an article container. Checked against className and id.
//
// TODO: Perhaps have these scale based on their odds of being quality?
export const NEGATIVE_SCORE_HINTS = [
  'adbox',
  'advert',
  'author',
  'bio',
  'bookmark',
  'bottom',
  'byline',
  'clear',
  'com-',
  'combx',
  'comment',
  'comment\\B',
  'contact',
  'copy',
  'credit',
  'crumb',
  'date',
  'deck',
  'excerpt',
  'featured', // tnr.com has a featured_content which throws us off
  'foot',
  'footer',
  'footnote',
  'graf',
  'head',
  'info',
  'infotext', // newscientist.com copyright
  'instapaper_ignore',
  'jump',
  'linebreak',
  'link',
  'masthead',
  'media',
  'meta',
  'modal',
  'outbrain', // slate.com junk
  'promo',
  'pr_', // autoblog - press release
  'related',
  'respond',
  'roundcontent', // lifehacker restricted content warning
  'scroll',
  'secondary',
  'share',
  'shopping',
  'shoutbox',
  'side',
  'sidebar',
  'sponsor',
  'stamp',
  'sub',
  'summary',
  'tags',
  'tools',
  'widget',
];
// The above list, joined into a matching regular expression
export const NEGATIVE_SCORE_RE = new RegExp(
  NEGATIVE_SCORE_HINTS.join('|'),
  'i'
);

// Match a digit. Pretty clear.
export const DIGIT_RE = new RegExp('[0-9]');

// Match 2 or more consecutive <br> tags
export const BR_TAGS_RE = new RegExp('(<br[^>]*>[ \n\r\t]*){2,}', 'i');

// Match 1 BR tag.
export const BR_TAG_RE = new RegExp('<br[^>]*>', 'i');

// A list of all of the block level tags known in HTML5 and below. Taken from
// http://bit.ly/qneNIT
export const BLOCK_LEVEL_TAGS = [
  'article',
  'aside',
  'blockquote',
  'body',
  'br',
  'button',
  'canvas',
  'caption',
  'col',
  'colgroup',
  'dd',
  'div',
  'dl',
  'dt',
  'embed',
  'fieldset',
  'figcaption',
  'figure',
  'footer',
  'form',
  'h1',
  'h2',
  'h3',
  'h4',
  'h5',
  'h6',
  'header',
  'hgroup',
  'hr',
  'li',
  'map',
  'object',
  'ol',
  'output',
  'p',
  'pre',
  'progress',
  'section',
  'table',
  'tbody',
  'textarea',
  'tfoot',
  'th',
  'thead',
  'tr',
  'ul',
  'video',
];
export const BLOCK_LEVEL_TAGS_RE = new RegExp(
  `^(${BLOCK_LEVEL_TAGS.join('|')})$`,
  'i'
);

// The removal is implemented as a blacklist and whitelist, this test finds
// blacklisted elements that aren't whitelisted. We do this all in one
// expression-both because it's only one pass, and because this skips the
// serialization for whitelisted nodes.
const candidatesBlacklist = UNLIKELY_CANDIDATES_BLACKLIST.join('|');
export const CANDIDATES_BLACKLIST = new RegExp(candidatesBlacklist, 'i');

const candidatesWhitelist = UNLIKELY_CANDIDATES_WHITELIST.join('|');
export const CANDIDATES_WHITELIST = new RegExp(candidatesWhitelist, 'i');

export const UNLIKELY_RE = new RegExp(
  `!(${candidatesWhitelist})|(${candidatesBlacklist})`,
  'i'
);

export const PARAGRAPH_SCORE_TAGS = new RegExp('^(p|li|span|pre)$', 'i');
export const CHILD_CONTENT_TAGS = new RegExp('^(td|blockquote|ol|ul|dl)$', 'i');
export const BAD_TAGS = new RegExp('^(address|form)$', 'i');

export const HTML_OR_BODY_RE = new RegExp('^(html|body)$', 'i');


<|start_filename|>src/extractors/generic/content/scoring/score-length.test.js<|end_filename|>
import assert from 'assert';

import { scoreLength } from './index';

describe('Scoring utils', () => {
  describe('scoreLength(textLength, tagName)', () => {
    it('returns 0 if length < 50 chars', () => {
      assert.equal(scoreLength(30), 0);
    });

    it('returns varying scores but maxes out at 3', () => {
      assert.equal(scoreLength(150), 1);
      assert.equal(scoreLength(199), 1.98);
      assert.equal(scoreLength(200), 2);
      assert.equal(scoreLength(250), 3);
      assert.equal(scoreLength(500), 3);
      assert.equal(scoreLength(1500), 3);
    });
  });
});


<|start_filename|>src/extractors/generic/content/scoring/score-commas.test.js<|end_filename|>
import assert from 'assert';

import { scoreCommas } from './index';

describe('Scoring utils', () => {
  describe('scoreCommas(text)', () => {
    it('returns 0 if text has no commas', () => {
      assert.equal(scoreCommas('Foo bar'), 0);
    });

    it('returns a point for every comma in the text', () => {
      assert.equal(scoreCommas('Foo, bar'), 1);
      assert.equal(scoreCommas('Foo, bar, baz'), 2);
      assert.equal(scoreCommas('Foo, bar, baz, bat'), 3);
    });
  });
});


<|start_filename|>src/utils/dom/link-density.js<|end_filename|>
export function textLength(text) {
  return text.trim().replace(/\s+/g, ' ').length;
}

// Determines what percentage of the text
// in a node is link text
// Takes a node, returns a float
export function linkDensity($node) {
  const totalTextLength = textLength($node.text());

  const linkText = $node.find('a').text();
  const linkLength = textLength(linkText);

  if (totalTextLength > 0) {
    return linkLength / totalTextLength;
  }
  if (totalTextLength === 0 && linkLength > 0) {
    return 1;
  }

  return 0;
}


<|start_filename|>src/extractors/custom/www.dmagazine.com/index.js<|end_filename|>
export const WwwDmagazineComExtractor = {
  domain: 'www.dmagazine.com',

  title: {
    selectors: ['h1.story__title'],
  },

  author: {
    selectors: ['.story__info .story__info__item:first-child'],
  },

  date_published: {
    selectors: [
      // enter selectors
      '.story__info',
    ],

    timezone: 'America/Chicago',
  },

  dek: {
    selectors: ['.story__subhead'],
  },

  lead_image_url: {
    selectors: [['article figure a:first-child', 'href']],
  },

  content: {
    selectors: ['.story__content'],

    // Is there anything in the content you selected that needs transformed
    // before it's consumable content? E.g., unusual lazy loaded images
    transforms: {},

    // Is there anything that is in the result that shouldn't be?
    // The clean selectors will remove anything that matches from
    // the result
    clean: [],
  },
};


<|start_filename|>src/extractors/generic/next-page-url/extractor.js<|end_filename|>
import URL from 'url';

import { articleBaseUrl, removeAnchor } from 'utils/text';
import scoreLinks from './scoring/score-links';

// Looks for and returns next page url
// for multi-page articles
const GenericNextPageUrlExtractor = {
  extract({ $, url, parsedUrl, previousUrls = [] }) {
    parsedUrl = parsedUrl || URL.parse(url);

    const articleUrl = removeAnchor(url);
    const baseUrl = articleBaseUrl(url, parsedUrl);

    const links = $('a[href]').toArray();

    const scoredLinks = scoreLinks({
      links,
      articleUrl,
      baseUrl,
      parsedUrl,
      $,
      previousUrls,
    });

    // If no links were scored, return null
    if (!scoredLinks) return null;

    // now that we've scored all possible pages,
    // find the biggest one.
    const topPage = Reflect.ownKeys(scoredLinks).reduce(
      (acc, link) => {
        const scoredLink = scoredLinks[link];
        return scoredLink.score > acc.score ? scoredLink : acc;
      },
      { score: -100 }
    );

    // If the score is less than 50, we're not confident enough to use it,
    // so we fail.
    if (topPage.score >= 50) {
      return topPage.href;
    }

    return null;
  },
};

export default GenericNextPageUrlExtractor;


<|start_filename|>src/utils/dom/fixtures/node-is-sufficient.js<|end_filename|>
const HTML = {
  tooShort: `
    <div class="foo bar">
      <p>This is too short</p>
    </div>
   `,
  longEnough: `
    <div class="foo bar">
      <p>
Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Aenean commodo ligula eget dolor. Aenean m
      </p>
    </div>
   `,
};

export default HTML;


<|start_filename|>src/extractors/custom/www.tmz.com/index.js<|end_filename|>
export const WwwTmzComExtractor = {
  domain: 'www.tmz.com',

  title: {
    selectors: ['.post-title-breadcrumb', 'h1', '.headline'],
  },

  author: '<NAME>',

  date_published: {
    selectors: ['.article-posted-date'],

    timezone: 'America/Los_Angeles',
  },

  dek: {
    selectors: [
      // enter selectors
    ],
  },

  lead_image_url: {
    selectors: [['meta[name="og:image"]', 'value']],
  },

  content: {
    selectors: ['.article-content', '.all-post-body'],

    // Is there anything in the content you selected that needs transformed
    // before it's consumable content? E.g., unusual lazy loaded images
    transforms: {},

    // Is there anything that is in the result that shouldn't be?
    // The clean selectors will remove anything that matches from
    // the result
    clean: ['.lightbox-link'],
  },
};


<|start_filename|>src/extractors/generic/next-page-url/scoring/utils/score-similarity.js<|end_filename|>
import difflib from 'difflib';

export default function scoreSimilarity(score, articleUrl, href) {
  // Do this last and only if we have a real candidate, because it's
  // potentially expensive computationally. Compare the link to this
  // URL using difflib to get the % similarity of these URLs. On a
  // sliding scale, subtract points from this link based on
  // similarity.
  if (score > 0) {
    const similarity = new difflib.SequenceMatcher(
      null,
      articleUrl,
      href
    ).ratio();
    // Subtract .1 from diff_percent when calculating modifier,
    // which means that if it's less than 10% different, we give a
    // bonus instead. Ex:
    //  3% different = +17.5 points
    // 10% different = 0 points
    // 20% different = -25 points
    const diffPercent = 1.0 - similarity;
    const diffModifier = -(250 * (diffPercent - 0.2));
    return score + diffModifier;
  }

  return 0;
}


<|start_filename|>src/extractors/generic/date-published/extractor.js<|end_filename|>
import { cleanDatePublished } from 'cleaners';
import { extractFromMeta, extractFromSelectors } from 'utils/dom';
import { extractFromUrl } from 'utils/text';

import {
  DATE_PUBLISHED_META_TAGS,
  DATE_PUBLISHED_SELECTORS,
  DATE_PUBLISHED_URL_RES,
} from './constants';

const GenericDatePublishedExtractor = {
  extract({ $, url, metaCache }) {
    let datePublished;
    // First, check to see if we have a matching meta tag
    // that we can make use of.
    // Don't try cleaning tags from this string
    datePublished = extractFromMeta(
      $,
      DATE_PUBLISHED_META_TAGS,
      metaCache,
      false
    );
    if (datePublished) return cleanDatePublished(datePublished);

    // Second, look through our selectors looking for potential
    // date_published's.
    datePublished = extractFromSelectors($, DATE_PUBLISHED_SELECTORS);
    if (datePublished) return cleanDatePublished(datePublished);

    // Lastly, look to see if a dately string exists in the URL
    datePublished = extractFromUrl(url, DATE_PUBLISHED_URL_RES);
    if (datePublished) return cleanDatePublished(datePublished);

    return null;
  },
};

export default GenericDatePublishedExtractor;


<|start_filename|>src/extractors/generic/content/scoring/get-or-init-score.js<|end_filename|>
import { getScore, scoreNode, getWeight, addToParent } from './index';

// gets and returns the score if it exists
// if not, initializes a score based on
// the node's tag type
export default function getOrInitScore($node, $, weightNodes = true) {
  let score = getScore($node);

  if (score) {
    return score;
  }

  score = scoreNode($node);

  if (weightNodes) {
    score += getWeight($node);
  }

  addToParent($node, $, score);

  return score;
}


<|start_filename|>src/cleaners/index.js<|end_filename|>
import cleanAuthor from './author';
import cleanImage from './lead-image-url';
import cleanDek from './dek';
import cleanDatePublished from './date-published';
import cleanContent from './content';
import cleanTitle from './title';

const Cleaners = {
  author: cleanAuthor,
  lead_image_url: cleanImage,
  dek: cleanDek,
  date_published: cleanDatePublished,
  content: cleanContent,
  title: cleanTitle,
};

export default Cleaners;

export { cleanAuthor };
export { cleanImage };
export { cleanDek };
export { cleanDatePublished };
export { cleanContent };
export { cleanTitle };
export { default as resolveSplitTitle } from './resolve-split-title';


<|start_filename|>src/extractors/generic/content/scoring/set-score.js<|end_filename|>
export default function setScore($node, $, score) {
  $node.attr('score', score);
  return $node;
}


<|start_filename|>src/extractors/generic/lead-image-url/fixtures/html.js<|end_filename|>
const HTML = {
  og: {
    test: `
      <html>
        <head>
          <meta name="og:image" value="http://example.com/lead.jpg">
        </head>
      </html>
    `,
    result: 'http://example.com/lead.jpg',
  },
  twitter: {
    test: `
      <html>
        <head>
          <meta name="twitter:image" value="http://example.com/lead.jpg">
        </head>
      </html>
    `,
    result: 'http://example.com/lead.jpg',
  },
  scoring: {
    test: `
      <div>
        <img src="http://example.com/sprite/abadpic.jpg" />
        <img src="http://example.com/upload/goodpic.jpg" />
        <img src="http://example.com/upload/whateverpic.png" />
      </div>
    `,
    result: 'http://example.com/upload/goodpic.jpg',
  },
  selectors: {
    test: `
      <div>
        <link rel="image_src" href="http://example.com/upload/goodpic.jpg">
      </div>
    `,
    result: 'http://example.com/upload/goodpic.jpg',
  },
};

export default HTML;


<|start_filename|>src/extractors/custom/www.popsugar.com/index.js<|end_filename|>
export const WwwPopsugarComExtractor = {
  domain: 'www.popsugar.com',

  title: {
    selectors: ['h2.post-title', 'title-text'],
  },

  author: {
    selectors: [['meta[name="article:author"]', 'value']],
  },

  date_published: {
    selectors: [['meta[name="article:published_time"]', 'value']],
  },

  lead_image_url: {
    selectors: [['meta[name="og:image"]', 'value']],
  },

  content: {
    selectors: ['#content'],

    // Is there anything in the content you selected that needs transformed
    // before it's consumable content? E.g., unusual lazy loaded images
    transforms: {},

    // Is there anything that is in the result that shouldn't be?
    // The clean selectors will remove anything that matches from
    // the result
    clean: ['.share-copy-title', '.post-tags', '.reactions'],
  },
};


<|start_filename|>src/extractors/detect-by-html.test.js<|end_filename|>
import assert from 'assert';
import cheerio from 'cheerio';

import detectByHtml from './detect-by-html';

describe('detectByHtml', () => {
  it('detects a medium post from the html', () => {
    const html = '<head><meta name="al:ios:app_name" value="Medium" /></head>';

    const $ = cheerio.load(html);

    assert.equal(detectByHtml($).domain, 'medium.com');
  });

  it('returns nothing if no match is found', () => {
    const html = '<div></div>';

    const $ = cheerio.load(html);

    assert.equal(detectByHtml($), null);
  });
});


<|start_filename|>src/utils/text/extract-from-url.js<|end_filename|>
// Given a node type to search for, and a list of regular expressions,
// look to see if this extraction can be found in the URL. Expects
// that each expression in r_list will return group(1) as the proper
// string to be cleaned.
// Only used for date_published currently.
export default function extractFromUrl(url, regexList) {
  const matchRe = regexList.find(re => re.test(url));
  if (matchRe) {
    return matchRe.exec(url)[1];
  }

  return null;
}


<|start_filename|>src/utils/dom/set-attrs.js<|end_filename|>
export default function setAttrs(node, attrs) {
  if (node.attribs) {
    node.attribs = attrs;
  } else if (node.attributes) {
    while (node.attributes.length > 0) {
      node.removeAttribute(node.attributes[0].name);
    }

    Reflect.ownKeys(attrs).forEach(key => {
      node.setAttribute(key, attrs[key]);
    });
  }

  return node;
}


<|start_filename|>src/extractors/generic/url/constants.js<|end_filename|>
export const CANONICAL_META_SELECTORS = ['og:url'];


<|start_filename|>src/cleaners/dek.test.js<|end_filename|>
import assert from 'assert';
import cheerio from 'cheerio';

import cleanDek from './dek';

describe('cleanDek(dekString, { $ })', () => {
  it('returns null if the dek is < 5 chars', () => {
    const $ = cheerio.load('<div></div>');
    assert.equal(cleanDek('Hi', { $ }), null);
  });

  it('returns null if the dek is > 1000 chars', () => {
    const $ = cheerio.load('<div></div>');
    const longDek =
      // generate a string that is 1,280 chars
      [0, 1, 2, 3, 4, 5, 6].reduce(acc => {
        acc += acc;
        return acc;
      }, '0123456789');
    assert.equal(cleanDek(longDek, { $ }), null);
  });

  it('strip html tags from the dek', () => {
    const $ = cheerio.load('<div></div>');
    const dek = 'This is a <em>very</em> important dek.';

    assert.equal(cleanDek(dek, { $ }), 'This is a very important dek.');
  });

  it('returns null if dek contains plain text link', () => {
    const $ = cheerio.load('<div></div>');
    const dek = 'This has this link http://example.com/foo/bar';

    assert.equal(cleanDek(dek, { $ }), null);
  });

  it('returns a normal dek as is', () => {
    const $ = cheerio.load('<div></div>');
    const dek = 'This is the dek';

    assert.equal(cleanDek(dek, { $ }), dek);
  });

  it('cleans extra whitespace', () => {
    const $ = cheerio.load('<div></div>');
    const dek = '    This is the dek   ';

    assert.equal(cleanDek(dek, { $ }), 'This is the dek');
  });

  it('returns null if the dek is the same as the excerpt', () => {
    const $ = cheerio.load('<div></div>');
    const excerpt = 'Hello to all of my friends';
    assert.equal(cleanDek(excerpt, { $, excerpt }), null);
  });
});


<|start_filename|>src/utils/dom/index.js<|end_filename|>
// DOM manipulation
export {
  default as stripUnlikelyCandidates,
} from './strip-unlikely-candidates';
export { default as brsToPs } from './brs-to-ps';
export { default as paragraphize } from './paragraphize';
export { default as convertToParagraphs } from './convert-to-paragraphs';
export { default as convertNodeTo } from './convert-node-to';
export { default as cleanImages } from './clean-images';
export { default as markToKeep } from './mark-to-keep';
export { default as stripJunkTags } from './strip-junk-tags';
export { default as cleanHOnes } from './clean-h-ones';
export { default as cleanAttributes } from './clean-attributes';
export { default as removeEmpty } from './remove-empty';
export { default as cleanTags } from './clean-tags';
export { default as cleanHeaders } from './clean-headers';
export { default as rewriteTopLevel } from './rewrite-top-level';
export { default as makeLinksAbsolute } from './make-links-absolute';
export { textLength, linkDensity } from './link-density';
export { default as extractFromMeta } from './extract-from-meta';
export { default as extractFromSelectors } from './extract-from-selectors';
export { default as stripTags } from './strip-tags';
export { default as withinComment } from './within-comment';
export { default as nodeIsSufficient } from './node-is-sufficient';
export { default as isWordpress } from './is-wordpress';
export { default as getAttrs } from './get-attrs';
export { default as setAttr } from './set-attr';
export { default as setAttrs } from './set-attrs';


<|start_filename|>src/extractors/generic/dek/extractor.test.js<|end_filename|>
import assert from 'assert';
import cheerio from 'cheerio';

import GenericDekExtractor from './extractor';

describe('GenericDekExtractor', () => {
  describe('extract({ $, metaCache })', () => {
    it('returns null if no dek can be found', () => {
      const $ = cheerio.load('<div></div>');
      const metaCache = [];
      const result = GenericDekExtractor.extract({ $, metaCache });

      assert.equal(result, null);
    });
  });
});


<|start_filename|>src/cleaners/lead-image-url.test.js<|end_filename|>
import assert from 'assert';

import clean from './lead-image-url';

describe('clean(leadImageUrl)', () => {
  it('returns the url if valid', () => {
    const url = 'https://example.com';
    assert.equal(clean(url), url);
  });

  it('returns null if the url is not valid', () => {
    const url = 'this is not a valid url';
    assert.equal(clean(url), null);
  });

  it('trims whitespace', () => {
    const url = '  https://example.com/foo/bar.jpg';
    assert.equal(clean(url), url.trim());
  });
});


<|start_filename|>src/extractors/custom/www.theverge.com/index.js<|end_filename|>
export const WwwThevergeComExtractor = {
  domain: 'www.theverge.com',

  supportedDomains: ['www.polygon.com'],

  title: {
    selectors: ['h1'],
  },

  author: {
    selectors: [['meta[name="author"]', 'value']],
  },

  date_published: {
    selectors: [['meta[name="article:published_time"]', 'value']],
  },

  dek: {
    selectors: ['h2.p-dek'],
  },

  lead_image_url: {
    selectors: [['meta[name="og:image"]', 'value']],
  },

  content: {
    selectors: [
      // feature template multi-match
      ['.c-entry-hero .e-image', '.c-entry-intro', '.c-entry-content'],
      // regular post multi-match
      ['.e-image--hero', '.c-entry-content'],
      // feature template fallback
      '.l-wrapper .l-feature',
      // regular post fallback
      'div.c-entry-content',
    ],

    // Transform lazy-loaded images
    transforms: {
      noscript: $node => {
        const $children = $node.children();
        if ($children.length === 1 && $children.get(0).tagName === 'img') {
          return 'span';
        }

        return null;
      },
    },

    // Is there anything that is in the result that shouldn't be?
    // The clean selectors will remove anything that matches from
    // the result
    clean: [
      '.aside',
      'img.c-dynamic-image', // images come from noscript transform
    ],
  },
};


<|start_filename|>src/extractors/custom/www.macrumors.com/index.js<|end_filename|>
export const WwwMacrumorsComExtractor = {
  domain: 'www.macrumors.com',

  title: {
    selectors: ['h1', 'h1.title'],
  },

  author: {
    selectors: ['.author-url'],
  },

  date_published: {
    selectors: ['.article .byline'],

    // Wednesday January 18, 2017 11:44 am PST
    format: 'dddd MMMM D, YYYY h:mm A zz',

    timezone: 'America/Los_Angeles',
  },

  dek: {
    selectors: [['meta[name="description"]', 'value']],
  },

  lead_image_url: {
    selectors: [['meta[name="og:image"]', 'value']],
  },

  content: {
    selectors: ['.article'],

    // Is there anything in the content you selected that needs transformed
    // before it's consumable content? E.g., unusual lazy loaded images
    transforms: {},

    // Is there anything that is in the result that shouldn't be?
    // The clean selectors will remove anything that matches from
    // the result
    clean: [],
  },
};


<|start_filename|>src/extractors/generic/content/scoring/score-commas.js<|end_filename|>
// return 1 for every comma in text
export default function scoreCommas(text) {
  return (text.match(/,/g) || []).length;
}


<|start_filename|>src/utils/dom/paragraphize.test.js<|end_filename|>
import assert from 'assert';
import cheerio from 'cheerio';

import { clean } from 'test-helpers';
import HTML from './fixtures/html';
import { paragraphize } from './index';

describe('Generic Extractor Utils', () => {
  describe('paragraphize(node)', () => {
    it('conversts a BR into P and moves inline contents to P tag after current parent', () => {
      const $ = cheerio.load(HTML.paragraphize.before);
      const node = $('br').get(0);

      // note: result here is not valid html; will handle elsewhere
      const result = paragraphize(node, $, true).html();

      assert.equal(clean(result), clean(HTML.paragraphize.after));
    });

    it('conversts a BR into P and stops when block element hit', () => {
      const $ = cheerio.load(HTML.paragraphizeBlock.before);
      const node = $('br').get(0);

      // note: result here is not valid html; will handle elsewhere
      const result = paragraphize(node, $, true).html();

      if ($.browser) {
        // small quirks in how jquery handles this vs. cheerio
        const html =
          '<p> Here is some text <p> Here is more text </p></p><div>And also this</div> <p></p>';
        assert.equal(clean(result), html);
      } else {
        assert.equal(clean(result), clean(HTML.paragraphizeBlock.after));
      }
    });
  });
});


<|start_filename|>src/utils/text/get-encoding.test.js<|end_filename|>
import assert from 'assert';
import cheerio from 'cheerio';

import getEncoding from './get-encoding';

// Tests are bypassed in the browser because it has an encoding
// A shim is used /src/shims/iconv-lite.js to decrease load size

describe('getEncoding(str)', () => {
  if (cheerio.browser) return;

  it('returns the encoding as a string', () => {
    const contentType = 'text/html; charset=iso-8859-15';
    assert.equal(getEncoding(contentType), 'iso-8859-15');
  });

  it('returns utf-8 as a default if no encoding found', () => {
    const contentType = 'text/html';
    assert.equal(getEncoding(contentType), 'utf-8');
  });

  it('returns utf-8 if there is an invalid encoding', () => {
    const contentType = 'text/html; charset=fake-charset';
    assert.equal(getEncoding(contentType), 'utf-8');
  });
});


<|start_filename|>src/extractors/generic/lead-image-url/score-image.js<|end_filename|>
import {
  POSITIVE_LEAD_IMAGE_URL_HINTS_RE,
  NEGATIVE_LEAD_IMAGE_URL_HINTS_RE,
  GIF_RE,
  JPG_RE,
} from './constants';

import { PHOTO_HINTS_RE } from '../content/scoring/constants';

function getSig($node) {
  return `${$node.attr('class') || ''} ${$node.attr('id') || ''}`;
}

// Scores image urls based on a variety of heuristics.
export function scoreImageUrl(url) {
  url = url.trim();
  let score = 0;

  if (POSITIVE_LEAD_IMAGE_URL_HINTS_RE.test(url)) {
    score += 20;
  }

  if (NEGATIVE_LEAD_IMAGE_URL_HINTS_RE.test(url)) {
    score -= 20;
  }

  // TODO: We might want to consider removing this as
  // gifs are much more common/popular than they once were
  if (GIF_RE.test(url)) {
    score -= 10;
  }

  if (JPG_RE.test(url)) {
    score += 10;
  }

  // PNGs are neutral.

  return score;
}

// Alt attribute usually means non-presentational image.
export function scoreAttr($img) {
  if ($img.attr('alt')) {
    return 5;
  }

  return 0;
}

// Look through our parent and grandparent for figure-like
// container elements, give a bonus if we find them
export function scoreByParents($img) {
  let score = 0;
  const $figParent = $img.parents('figure').first();

  if ($figParent.length === 1) {
    score += 25;
  }

  const $parent = $img.parent();
  let $gParent;
  if ($parent.length === 1) {
    $gParent = $parent.parent();
  }

  [$parent, $gParent].forEach($node => {
    if (PHOTO_HINTS_RE.test(getSig($node))) {
      score += 15;
    }
  });

  return score;
}

// Look at our immediate sibling and see if it looks like it's a
// caption. Bonus if so.
export function scoreBySibling($img) {
  let score = 0;
  const $sibling = $img.next();
  const sibling = $sibling.get(0);

  if (sibling && sibling.tagName.toLowerCase() === 'figcaption') {
    score += 25;
  }

  if (PHOTO_HINTS_RE.test(getSig($sibling))) {
    score += 15;
  }

  return score;
}

export function scoreByDimensions($img) {
  let score = 0;

  const width = parseFloat($img.attr('width'));
  const height = parseFloat($img.attr('height'));
  const src = $img.attr('src');

  // Penalty for skinny images
  if (width && width <= 50) {
    score -= 50;
  }

  // Penalty for short images
  if (height && height <= 50) {
    score -= 50;
  }

  if (width && height && !src.includes('sprite')) {
    const area = width * height;
    if (area < 5000) {
      // Smaller than 50 x 100
      score -= 100;
    } else {
      score += Math.round(area / 1000);
    }
  }

  return score;
}

export function scoreByPosition($imgs, index) {
  return $imgs.length / 2 - index;
}


<|start_filename|>src/cleaners/fixtures/html.js<|end_filename|>
const HTML = {
  docWithH1: '<div><h1>This Is the Real Title</h1></div>',
  docWith2H1s: `
    <div>
      <h1>This Is the Real Title</h1>
      <h1>This Is the Real Title</h1>
    </div>
  `,
  docWithTagsInH1: {
    before: '<div><h1>This Is the <em>Real</em> Title</h1></div>',
    after: 'This Is the Real Title',
  },
};

export default HTML;


<|start_filename|>src/extractors/custom/fusion.net/index.js<|end_filename|>
export const FusionNetExtractor = {
  domain: 'fusion.net',

  title: {
    selectors: ['.post-title', '.single-title', '.headline'],
  },

  author: {
    selectors: ['.show-for-medium .byline'],
  },

  date_published: {
    selectors: [['time.local-time', 'datetime']],
  },

  dek: {
    selectors: [
      // enter selectors
    ],
  },

  lead_image_url: {
    selectors: [['meta[name="og:image"]', 'value']],
  },

  content: {
    selectors: [
      ['.post-featured-media', '.article-content'],
      '.article-content',
    ],

    // Is there anything in the content you selected that needs transformed
    // before it's consumable content? E.g., unusual lazy loaded images
    transforms: {
      '.fusion-youtube-oembed': 'figure',
    },

    // Is there anything that is in the result that shouldn't be?
    // The clean selectors will remove anything that matches from
    // the result
    clean: [],
  },
};


<|start_filename|>src/extractors/custom/news.mynavi.jp/index.js<|end_filename|>
export const NewsMynaviJpExtractor = {
  domain: 'news.mynavi.jp',

  title: {
    selectors: [['meta[name="og:title"]', 'value']],
  },

  author: {
    selectors: ['main div.article-author a.article-author__name'],
  },

  date_published: {
    selectors: [['meta[name="article:published_time"]', 'value']],
  },

  dek: {
    selectors: [['meta[name="og:description"]', 'value']],
  },

  lead_image_url: {
    selectors: [['meta[name="og:image"]', 'value']],
  },

  content: {
    selectors: ['main article div'],

    // Is there anything in the content you selected that needs transformed
    // before it's consumable content? E.g., unusual lazy loaded images
    transforms: {
      img: $node => {
        const src = $node.attr('data-original');
        if (src !== '') {
          $node.attr('src', src);
        }
      },
    },

    // Is there anything that is in the result that shouldn't be?
    // The clean selectors will remove anything that matches from
    // the result
    clean: [],
  },
};


<|start_filename|>src/utils/text/article-base-url.test.js<|end_filename|>
import assert from 'assert';

import articleBaseUrl from './article-base-url';

describe('articleBaseUrl(url, parsedUrl)', () => {
  it('returns the base url of a paginated url', () => {
    const url = 'http://example.com/foo/bar/wow-cool/page=10';
    const cleaned = 'http://example.com/foo/bar/wow-cool';

    assert.equal(articleBaseUrl(url), cleaned);
  });

  it('returns same url if url has no pagination info', () => {
    const url = 'http://example.com/foo/bar/wow-cool/';
    const cleaned = 'http://example.com/foo/bar/wow-cool';

    assert.equal(articleBaseUrl(url), cleaned);
  });
});


<|start_filename|>src/extractors/generic/author/constants.js<|end_filename|>
// An ordered list of meta tag names that denote likely article authors. All
// attributes should be lowercase for faster case-insensitive matching. From
// most distinct to least distinct.
//
// Note: "author" is too often the -developer- of the page, so it is not
// added here.
export const AUTHOR_META_TAGS = [
  'byl',
  'clmst',
  'dc.author',
  'dcsext.author',
  'dc.creator',
  'rbauthors',
  'authors',
];

export const AUTHOR_MAX_LENGTH = 300;

// An ordered list of XPath Selectors to find likely article authors. From
// most explicit to least explicit.
//
// Note - this does not use classes like CSS. This checks to see if the string
// exists in the className, which is not as accurate as .className (which
// splits on spaces/endlines), but for our purposes it's close enough. The
// speed tradeoff is worth the accuracy hit.
export const AUTHOR_SELECTORS = [
  '.entry .entry-author',
  '.author.vcard .fn',
  '.author .vcard .fn',
  '.byline.vcard .fn',
  '.byline .vcard .fn',
  '.byline .by .author',
  '.byline .by',
  '.byline .author',
  '.post-author.vcard',
  '.post-author .vcard',
  'a[rel=author]',
  '#by_author',
  '.by_author',
  '#entryAuthor',
  '.entryAuthor',
  '.byline a[href*=author]',
  '#author .authorname',
  '.author .authorname',
  '#author',
  '.author',
  '.articleauthor',
  '.ArticleAuthor',
  '.byline',
];

// An ordered list of Selectors to find likely article authors, with
// regular expression for content.
const bylineRe = /^[\n\s]*By/i;
export const BYLINE_SELECTORS_RE = [
  ['#byline', bylineRe],
  ['.byline', bylineRe],
];


<|start_filename|>src/extractors/custom/newrepublic.com/index.js<|end_filename|>
export const NewrepublicComExtractor = {
  domain: 'newrepublic.com',

  title: {
    selectors: ['h1.article-headline', '.minutes-primary h1.minute-title'],
  },

  author: {
    selectors: ['div.author-list', '.minutes-primary h3.minute-byline'],
  },

  date_published: {
    selectors: [['meta[name="article:published_time"]', 'value']],

    timezone: 'America/New_York',
  },

  dek: {
    selectors: ['h2.article-subhead'],
  },

  lead_image_url: {
    selectors: [['meta[name="og:image"]', 'value']],
  },

  content: {
    selectors: [
      ['.article-cover', 'div.content-body'],
      ['.minute-image', '.minutes-primary div.content-body'],
    ],

    // Is there anything in the content you selected that needs transformed
    // before it's consumable content? E.g., unusual lazy loaded images
    transforms: {},

    // Is there anything that is in the result that shouldn't be?
    // The clean selectors will remove anything that matches from
    // the result
    clean: ['aside'],
  },
};


<|start_filename|>src/extractors/custom/news.nationalgeographic.com/index.test.js<|end_filename|>
import assert from 'assert';
import URL from 'url';
import cheerio from 'cheerio';

import Mercury from 'mercury';
import getExtractor from 'extractors/get-extractor';
import { excerptContent } from 'utils/text';

const fs = require('fs');

describe('NewsNationalgeographicComExtractor', () => {
  describe('initial test case', () => {
    let result;
    let url;
    beforeAll(() => {
      url =
        'http://news.nationalgeographic.com/energy/2015/08/150803-gender-bias-affects-office-heating-cooling-temperatures/';
      const html = fs.readFileSync(
        './fixtures/news.nationalgeographic.com/1481919545107.html'
      );
      result = Mercury.parse(url, { html, fallback: false });
    });

    it('is selected properly', () => {
      // This test should be passing by default.
      // It sanity checks that the correct parser
      // is being selected for URLs from this domain
      const extractor = getExtractor(url);
      assert.equal(extractor.domain, URL.parse(url).hostname);
    });

    it('returns the title', async () => {
      // To pass this test, fill out the title selector
      // in ./src/extractors/custom/news.nationalgeographic.com/index.js.
      const { title } = await result;

      // Update these values with the expected values from
      // the article.
      assert.equal(
        title,
        'Here’s Why Your Office May Be Too Hot or Cold: Gender Bias'
      );
    });

    it('returns the date_published', async () => {
      // To pass this test, fill out the date_published selector
      // in ./src/extractors/custom/news.nationalgeographic.com/index.js.
      const { date_published } = await result;

      // Update these values with the expected values from
      // the article.
      assert.equal(date_published, '2015-08-03T17:45:00.000Z');
    });

    it('returns the dek', async () => {
      // To pass this test, fill out the dek selector
      // in ./src/extractors/custom/news.nationalgeographic.com/index.js.
      const { dek } = await result;

      // Update these values with the expected values from
      // the article.
      assert.equal(
        dek,
        'Do you argue about the temperature in your office or home? Find out what often decides it, and tell us your preference.'
      );
    });

    it('returns the lead_image_url', async () => {
      // To pass this test, fill out the lead_image_url selector
      // in ./src/extractors/custom/news.nationalgeographic.com/index.js.
      const { lead_image_url } = await result;

      // Update these values with the expected values from
      // the article.
      assert.equal(
        lead_image_url,
        'http://news.nationalgeographic.com/content/dam/news/2015/08/03/temperaturegenderbias/02tempgenderbias.ngsversion.1438795800319.jpg'
      );
    });

    it('returns the content', async () => {
      // To pass this test, fill out the content selector
      // in ./src/extractors/custom/news.nationalgeographic.com/index.js.
      // You may also want to make use of the clean and transform
      // options.
      const { content } = await result;

      const $ = cheerio.load(content || '');

      const first13 = excerptContent(
        $('*')
          .first()
          .text(),
        13
      );

      // Update these values with the expected values from
      // the article.
      assert.equal(
        first13,
        'Many couples fight about it at home. No, it’s not money, sex, or'
      );
    });
  });
});


<|start_filename|>src/extractors/generic/excerpt/extractor.js<|end_filename|>
import ellipsize from 'ellipsize';

import { extractFromMeta, stripTags } from 'utils/dom';

import { EXCERPT_META_SELECTORS } from './constants';

export function clean(content, $, maxLength = 200) {
  content = content.replace(/[\s\n]+/g, ' ').trim();
  return ellipsize(content, maxLength, { ellipse: '&hellip;' });
}

const GenericExcerptExtractor = {
  extract({ $, content, metaCache }) {
    const excerpt = extractFromMeta($, EXCERPT_META_SELECTORS, metaCache);
    if (excerpt) {
      return clean(stripTags(excerpt, $));
    }
    // Fall back to excerpting from the extracted content
    const maxLength = 200;
    const shortContent = content.slice(0, maxLength * 5);
    return clean($(shortContent).text(), $, maxLength);
  },
};

export default GenericExcerptExtractor;


<|start_filename|>src/extractors/collect-all-pages.js<|end_filename|>
import { removeAnchor } from 'utils/text';
import RootExtractor from 'extractors/root-extractor';
import GenericExtractor from 'extractors/generic';
import Resource from 'resource';

export default async function collectAllPages({
  next_page_url,
  html,
  $,
  metaCache,
  result,
  Extractor,
  title,
  url,
}) {
  // At this point, we've fetched just the first page
  let pages = 1;
  const previousUrls = [removeAnchor(url)];

  // If we've gone over 26 pages, something has
  // likely gone wrong.
  while (next_page_url && pages < 26) {
    pages += 1;
    // eslint-disable-next-line no-await-in-loop
    $ = await Resource.create(next_page_url);
    html = $.html();

    const extractorOpts = {
      url: next_page_url,
      html,
      $,
      metaCache,
      contentOnly: true,
      extractedTitle: title,
      previousUrls,
    };

    const nextPageResult = RootExtractor.extract(Extractor, extractorOpts);

    previousUrls.push(next_page_url);
    result = {
      ...result,
      content: `${result.content}<hr><h4>Page ${pages}</h4>${
        nextPageResult.content
      }`,
    };

    // eslint-disable-next-line prefer-destructuring
    next_page_url = nextPageResult.next_page_url;
  }

  const word_count = GenericExtractor.word_count({
    content: `<div>${result.content}</div>`,
  });
  return {
    ...result,
    total_pages: pages,
    pages_rendered: pages,
    word_count,
  };
}


<|start_filename|>src/utils/dom/clean-headers.js<|end_filename|>
import { getWeight } from 'extractors/generic/content/scoring';

import { HEADER_TAG_LIST } from './constants';
import { normalizeSpaces } from '../text';

export default function cleanHeaders($article, $, title = '') {
  $(HEADER_TAG_LIST, $article).each((index, header) => {
    const $header = $(header);
    // Remove any headers that appear before all other p tags in the
    // document. This probably means that it was part of the title, a
    // subtitle or something else extraneous like a datestamp or byline,
    // all of which should be handled by other metadata handling.
    if ($($header, $article).prevAll('p').length === 0) {
      return $header.remove();
    }

    // Remove any headers that match the title exactly.
    if (normalizeSpaces($(header).text()) === title) {
      return $header.remove();
    }

    // If this header has a negative weight, it's probably junk.
    // Get rid of it.
    if (getWeight($(header)) < 0) {
      return $header.remove();
    }

    return $header;
  });

  return $;
}


<|start_filename|>src/extractors/generic/dek/extractor.js<|end_filename|>
// Currently there is only one selector for
// deks. We should simply return null here
// until we have a more robust generic option.
// Below is the original source for this, for reference.
const GenericDekExtractor = {
  extract() {
    return null;
  },
};

export default GenericDekExtractor;


<|start_filename|>scripts/proxy-browser-test.js<|end_filename|>
/* eslint-disable */
const express = require('express'); // eslint-disable-line import/no-extraneous-dependencies
const request = require('request');

const app = express();
var server;

const start = () => {
  app.use('/:url', (req, res) => {
    const url = req.originalUrl.slice(1);

    const options = {
      url,
      // Don't set encoding; fixes issues
      // w/gzipped responses
      encoding: null,
      // Accept cookies
      jar: true,
      // Accept and decode gzip
      gzip: true,
      // Follow any redirect
      followAllRedirects: true,
    };
    req.pipe(request(options)).pipe(res);
  });

  server = app.listen(process.env.PORT || 3000);
};

const stop = () => {
  server && server.close();
};

start();
require('child_process').exec(
  'node ./node_modules/karma/bin/karma start ./scripts/karma.conf.js' +
    (process.env.CI ? ' --CI' : ''),
  { stdio: [0, 1, 2] },
  (err, stdout) => {
    if (err) {
      console.log('stdout', stdout);
      process.exit(1);
    } else {
      console.log('stdout', stdout);
    }
    stop();
  }
);


<|start_filename|>src/utils/dom/mark-to-keep.test.js<|end_filename|>
import cheerio from 'cheerio';
import assert from 'assert';

import { assertClean } from 'test-helpers';

import HTML from './fixtures/html';
import { markToKeep } from './index';
import { KEEP_CLASS } from './constants';

describe('markToKeep($)', () => {
  it('marks elements that should be kept', () => {
    const $ = cheerio.load(HTML.marksYouTube.before);

    const result = markToKeep($('*').first(), $);

    assert.equal(result('iframe.mercury-parser-keep').length, 2);

    if (!$.browser) {
      assertClean(result.html(), HTML.marksYouTube.after);
    }
  });

  it('marks same-domain elements to keep', () => {
    const html =
      '<div><iframe src="https://medium.com/foo/bar"></iframe></div>';
    const $ = cheerio.load(html);

    const result = markToKeep($('*').first(), $, 'https://medium.com/foo');

    const keptHtml = `<div><iframe src="https://medium.com/foo/bar" class="${KEEP_CLASS}"></iframe></div>`;
    assertClean(result.html(), keptHtml);
  });
});


<|start_filename|>src/extractors/generic/next-page-url/scoring/utils/score-extraneous-links.js<|end_filename|>
import { EXTRANEOUS_LINK_HINTS_RE } from '../constants';

export default function scoreExtraneousLinks(href) {
  // If the URL itself contains extraneous values, give a penalty.
  if (EXTRANEOUS_LINK_HINTS_RE.test(href)) {
    return -25;
  }

  return 0;
}


<|start_filename|>src/utils/text/article-base-url.js<|end_filename|>
import URL from 'url';

import {
  HAS_ALPHA_RE,
  IS_ALPHA_RE,
  IS_DIGIT_RE,
  PAGE_IN_HREF_RE,
} from './constants';

function isGoodSegment(segment, index, firstSegmentHasLetters) {
  let goodSegment = true;

  // If this is purely a number, and it's the first or second
  // url_segment, it's probably a page number. Remove it.
  if (index < 2 && IS_DIGIT_RE.test(segment) && segment.length < 3) {
    goodSegment = true;
  }

  // If this is the first url_segment and it's just "index",
  // remove it
  if (index === 0 && segment.toLowerCase() === 'index') {
    goodSegment = false;
  }

  // If our first or second url_segment is smaller than 3 characters,
  // and the first url_segment had no alphas, remove it.
  if (index < 2 && segment.length < 3 && !firstSegmentHasLetters) {
    goodSegment = false;
  }

  return goodSegment;
}

// Take a URL, and return the article base of said URL. That is, no
// pagination data exists in it. Useful for comparing to other links
// that might have pagination data within them.
export default function articleBaseUrl(url, parsed) {
  const parsedUrl = parsed || URL.parse(url);
  const { protocol, host, path } = parsedUrl;

  let firstSegmentHasLetters = false;
  const cleanedSegments = path
    .split('/')
    .reverse()
    .reduce((acc, rawSegment, index) => {
      let segment = rawSegment;

      // Split off and save anything that looks like a file type.
      if (segment.includes('.')) {
        const [possibleSegment, fileExt] = segment.split('.');
        if (IS_ALPHA_RE.test(fileExt)) {
          segment = possibleSegment;
        }
      }

      // If our first or second segment has anything looking like a page
      // number, remove it.
      if (PAGE_IN_HREF_RE.test(segment) && index < 2) {
        segment = segment.replace(PAGE_IN_HREF_RE, '');
      }

      // If we're on the first segment, check to see if we have any
      // characters in it. The first segment is actually the last bit of
      // the URL, and this will be helpful to determine if we're on a URL
      // segment that looks like "/2/" for example.
      if (index === 0) {
        firstSegmentHasLetters = HAS_ALPHA_RE.test(segment);
      }

      // If it's not marked for deletion, push it to cleaned_segments.
      if (isGoodSegment(segment, index, firstSegmentHasLetters)) {
        acc.push(segment);
      }

      return acc;
    }, []);

  return `${protocol}//${host}${cleanedSegments.reverse().join('/')}`;
}


<|start_filename|>src/extractors/custom/www.huffingtonpost.com/index.js<|end_filename|>
export const WwwHuffingtonpostComExtractor = {
  domain: 'www.huffingtonpost.com',

  title: {
    selectors: ['h1.headline__title'],
  },

  author: {
    selectors: ['span.author-card__details__name'],
  },

  date_published: {
    selectors: [
      ['meta[name="article:modified_time"]', 'value'],
      ['meta[name="article:published_time"]', 'value'],
    ],
  },

  dek: {
    selectors: ['h2.headline__subtitle'],
  },

  lead_image_url: {
    selectors: [['meta[name="og:image"]', 'value']],
  },

  content: {
    selectors: ['div.entry__body'],

    defaultCleaner: false,

    // Is there anything in the content you selected that needs transformed
    // before it's consumable content? E.g., unusual lazy loaded images
    transforms: {},

    // Is there anything that is in the result that shouldn't be?
    // The clean selectors will remove anything that matches from
    // the result
    clean: [
      '.pull-quote',
      '.tag-cloud',
      '.embed-asset',
      '.below-entry',
      '.entry-corrections',
      '#suggested-story',
    ],
  },
};


<|start_filename|>src/extractors/generic/content/extractor.test.js<|end_filename|>
import assert from 'assert';

import { clean } from 'test-helpers';

import GenericContentExtractor from './extractor';

const fs = require('fs');

describe('GenericContentExtractor', () => {
  describe('extract($, html, opts)', () => {
    it('extracts html and returns the article', () => {
      const html = fs.readFileSync('./fixtures/vulture.html', 'utf-8');
      const result = clean(
        GenericContentExtractor.extract({
          $: null,
          html,
          url:
            'http://www.vulture.com/2016/08/dc-comics-greg-berlanti-c-v-r.html',
        })
      );

      assert(typeof result, 'string');
    });
  });
});


<|start_filename|>src/extractors/generic/content/scoring/get-or-init-score.test.js<|end_filename|>
import assert from 'assert';
import cheerio from 'cheerio';

import HTML from './fixtures/html';
import { getOrInitScore, getScore } from './index';

describe('getOrInitScore(node, $)', () => {
  describe('when score set', () => {
    it("returns score if node's score already set", () => {
      const html = '<p score="40">Foo</p>';
      const $ = cheerio.load(html);
      const node = $('p').first();

      const score = getOrInitScore(node, $);

      assert.equal(score, 40);
    });
  });

  describe('when no score set', () => {
    it('returns 0 if no class/id and text < 25 chars', () => {
      const html = '<p>Foo</p>';
      const $ = cheerio.load(html);
      const node = $('p').first();

      const score = getOrInitScore(node, $);

      assert.equal(score, 0);
    });

    it('returns score if no class/id and has commas/length', () => {
      const $ = cheerio.load(HTML.score19);
      const node = $('p').first();

      const score = getOrInitScore(node, $);

      assert.equal(score, 19);
    });

    it('returns greater score if weighted class/id is set', () => {
      const $ = cheerio.load(HTML.score44);
      const node = $('p').first();

      const score = getOrInitScore(node, $);

      assert.equal(score, 44);
    });

    it('gives 1/4 of its score to its parent', () => {
      const $ = cheerio.load(HTML.score44Parent);
      const node = $('p').first();

      getOrInitScore(node, $);

      assert.equal(getScore(node.parent()), 16);
    });
  });
});


<|start_filename|>src/utils/dom/strip-tags.js<|end_filename|>
// strips all tags from a string of text
export default function stripTags(text, $) {
  // Wrapping text in html element prevents errors when text
  // has no html
  const cleanText = $(`<span>${text}</span>`).text();
  return cleanText === '' ? text : cleanText;
}


<|start_filename|>scripts/templates/index.js<|end_filename|>
import insertValues from './insert-values';

const bodyPattern = /^\n([\s\S]+)\s{2}$/gm;
const trailingWhitespace = /\s+$/;

export default function template(strings, ...values) {
  const compiled = insertValues(strings, ...values);
  let [body] = compiled.match(bodyPattern) || [];
  let indentLevel = /^\s{0,4}(.+)$/g;

  if (!body) {
    body = compiled;
    indentLevel = /^\s{0,2}(.+)$/g;
  }

  return body
    .split('\n')
    .slice(1)
    .map(line => {
      line = line.replace(indentLevel, '$1');

      if (trailingWhitespace.test(line)) {
        line = line.replace(trailingWhitespace, '');
      }

      return line;
    })
    .join('\n');
}


<|start_filename|>src/utils/dom/strip-tags.test.js<|end_filename|>
import assert from 'assert';
import cheerio from 'cheerio';

import stripTags from './strip-tags';

describe('stripTags(title, $)', () => {
  it('strips tags from a string of text', () => {
    const $ = cheerio.load('<div></div>');

    const result = stripTags('What a <em>Wonderful</em> Day', $);

    assert.equal(result, 'What a Wonderful Day');
  });

  it('returns the original text if no tags found', () => {
    const $ = cheerio.load('<div></div>');

    const result = stripTags('What a Wonderful Day', $);

    assert.equal(result, 'What a Wonderful Day');
  });
});


<|start_filename|>src/extractors/generic/title/extractor.js<|end_filename|>
import { cleanTitle } from 'cleaners';
import { extractFromMeta, extractFromSelectors } from 'utils/dom';

import {
  STRONG_TITLE_META_TAGS,
  WEAK_TITLE_META_TAGS,
  STRONG_TITLE_SELECTORS,
  WEAK_TITLE_SELECTORS,
} from './constants';

const GenericTitleExtractor = {
  extract({ $, url, metaCache }) {
    // First, check to see if we have a matching meta tag that we can make
    // use of that is strongly associated with the headline.
    let title;

    title = extractFromMeta($, STRONG_TITLE_META_TAGS, metaCache);
    if (title) return cleanTitle(title, { url, $ });

    // Second, look through our content selectors for the most likely
    // article title that is strongly associated with the headline.
    title = extractFromSelectors($, STRONG_TITLE_SELECTORS);
    if (title) return cleanTitle(title, { url, $ });

    // Third, check for weaker meta tags that may match.
    title = extractFromMeta($, WEAK_TITLE_META_TAGS, metaCache);
    if (title) return cleanTitle(title, { url, $ });

    // Last, look for weaker selector tags that may match.
    title = extractFromSelectors($, WEAK_TITLE_SELECTORS);
    if (title) return cleanTitle(title, { url, $ });

    // If no matches, return an empty string
    return '';
  },
};

export default GenericTitleExtractor;


<|start_filename|>src/extractors/custom/www.youtube.com/index.js<|end_filename|>
export const WwwYoutubeComExtractor = {
  domain: 'www.youtube.com',

  title: {
    selectors: ['.watch-title', 'h1.watch-title-container'],
  },

  author: {
    selectors: ['.yt-user-info'],
  },

  date_published: {
    selectors: [['meta[itemProp="datePublished"]', 'value']],

    timezone: 'GMT',
  },

  dek: {
    selectors: [
      // enter selectors
    ],
  },

  lead_image_url: {
    selectors: [['meta[name="og:image"]', 'value']],
  },

  content: {
    defaultCleaner: false,

    selectors: [['#player-api', '#eow-description']],

    // Is there anything in the content you selected that needs transformed
    // before it's consumable content? E.g., unusual lazy loaded images
    transforms: {
      '#player-api': ($node, $) => {
        const videoId = $('meta[itemProp="videoId"]').attr('value');
        $node.html(`
          <iframe src="https://www.youtube.com/embed/${videoId}" frameborder="0" allowfullscreen></iframe>`);
      },
    },

    // Is there anything that is in the result that shouldn't be?
    // The clean selectors will remove anything that matches from
    // the result
    clean: [],
  },
};


<|start_filename|>src/extractors/generic/author/extractor.test.js<|end_filename|>
import assert from 'assert';
import cheerio from 'cheerio';

import HTML from './fixtures/html';
import GenericAuthorExtractor from './extractor';

describe('GenericAuthorExtractor', () => {
  describe('extract($, cachedMeta)', () => {
    it('extracts author from meta tags', () => {
      const $ = cheerio.load(HTML.authorMeta.test);
      const result = GenericAuthorExtractor.extract({
        $,
        metaCache: ['dc.author', 'something-else'],
      });

      assert.equal(result, HTML.authorMeta.result);
    });

    it('extracts author from author selectors', () => {
      const $ = cheerio.load(HTML.authorSelectors.test);
      const result = GenericAuthorExtractor.extract({
        $,
        metaCache: ['dc.author', 'something-else'],
      });

      assert.equal(result, HTML.authorSelectors.result);
    });

    it('extracts author with regex selectors', () => {
      const $ = cheerio.load(HTML.authorRegSelectors.test);
      const result = GenericAuthorExtractor.extract({
        $,
        metaCache: ['dc.author', 'something-else'],
      });

      assert.equal(result, HTML.authorRegSelectors.result);
    });

    it('returns null if no author found', () => {
      const $ = cheerio.load('<div></div>');
      const result = GenericAuthorExtractor.extract({
        $,
        metaCache: ['dc.author', 'something-else'],
      });

      assert.equal(result, null);
    });
  });
});


<|start_filename|>src/utils/dom/clean-attributes.test.js<|end_filename|>
import cheerio from 'cheerio';

import { assertClean } from 'test-helpers';

import HTML from './fixtures/html';
import { cleanAttributes } from './index';

describe('cleanAttributes($)', () => {
  it('removes style attributes from nodes', () => {
    const $ = cheerio.load(HTML.removeStyle.before);

    const result = cleanAttributes($('*').first(), $);
    assertClean($.html(result), HTML.removeStyle.after);
  });

  it('removes align attributes from nodes', () => {
    const $ = cheerio.load(HTML.removeAlign.before);

    const result = cleanAttributes($('*').first(), $);
    assertClean($.html(result), HTML.removeAlign.after);
  });
});


<|start_filename|>src/extractors/generic/lead-image-url/score-image.test.js<|end_filename|>
import assert from 'assert';
import cheerio from 'cheerio';

import {
  scoreImageUrl,
  scoreAttr,
  scoreByParents,
  scoreBySibling,
  scoreByDimensions,
  scoreByPosition,
} from './score-image';

describe('scoreImageUrlUrl(url)', () => {
  it('gets 20 points for a positive lead img hint', () => {
    const url = 'http://example.com/upload/img.png';

    assert.equal(scoreImageUrl(url), 20);
  });

  it('loses 20 points for a negative lead img hint', () => {
    const url = 'http://example.com/sprite/foo/bar.png';

    assert.equal(scoreImageUrl(url), -20);
  });

  it('loses 10 points for a gif', () => {
    const url = 'http://example.com/foo/bar.gif';

    assert.equal(scoreImageUrl(url), -10);

    const url2 = 'http://example.com/foogif/bar';

    assert.equal(scoreImageUrl(url2), 0);
  });

  it('gains 10 points for a jpg', () => {
    const url = 'http://example.com/foo/bar.jpg';
    assert.equal(scoreImageUrl(url), 10);

    const url2 = 'http://example.com/foo/bar.jpeg';
    assert.equal(scoreImageUrl(url2), 10);

    const url3 = 'http://example.com/foojpg/bar';
    assert.equal(scoreImageUrl(url3), 0);

    const url4 = 'http://example.com/foo.jpg?bar=baz';
    assert.equal(scoreImageUrl(url4), 10);
  });
});

describe('scoreAttr($img)', () => {
  it('gets 5 points if the img node has an alt attribute', () => {
    const $ = cheerio.load('<div><img alt="Wow" /></div>');
    const $img = $('img').first();

    assert.equal(scoreAttr($img), 5);
  });

  it('gets 0 points if the img node has an alt attribute', () => {
    const $ = cheerio.load('<div><img /></div>');
    const $img = $('img').first();

    assert.equal(scoreAttr($img), 0);
  });
});

describe('scoreByParents($img)', () => {
  it('gets 25 points if it has a figure parent', () => {
    const $ = cheerio.load(
      `<div>
          <figure>
            <div>
              <img alt="Wow" />
            </div>
          </figure>
        </div>`
    );
    const $img = $('img').first();

    assert.equal(scoreByParents($img), 25);
  });

  it('gets 0 points if the img has no figure parent', () => {
    const $ = cheerio.load('<div><img /></div>');
    const $img = $('img').first();

    assert.equal(scoreByParents($img), 0);
  });

  it('gets 15 points if parent or gparent has photo hints', () => {
    const $ = cheerio.load(
      `<div>
          <div class="figure">
            <div>
              <img alt="Wow" />
            </div>
          </div>
        </div>`
    );
    const $img = $('img').first();

    assert.equal(scoreByParents($img), 15);
  });
});

describe('scoreBySibling($img)', () => {
  it('gets 25 points if its sibling is figcaption', () => {
    const $ = cheerio.load(
      `
      <div>
        <img />
        <figcaption>Wow</figcaption>
      </div>
      `
    );
    const $img = $('img').first();

    assert.equal(scoreBySibling($img), 25);
  });

  it('gets 15 points if its sibling has photo hints', () => {
    const $ = cheerio.load(
      `<div>
          <div>
              <img alt="Wow" />
              <div class="caption">
                Wow
              </div>
          </div>
        </div>`
    );
    const $img = $('img').first();

    assert.equal(scoreBySibling($img), 15);
  });
});

describe('scoreByDimensions($img)', () => {
  it('penalizes skinny images', () => {
    const $ = cheerio.load(
      `
      <div>
        <img width="10" />
      </div>
      `
    );
    const $img = $('img').first();

    assert.equal(scoreByDimensions($img), -50);
  });

  it('penalizes short images', () => {
    const $ = cheerio.load(
      `
      <div>
        <img height="10" />
      </div>
      `
    );
    const $img = $('img').first();

    assert.equal(scoreByDimensions($img), -50);
  });

  it('ignores sprites', () => {
    const $ = cheerio.load(
      `
      <div>
        <img src="/sprite/etc/foo.png" width="1000" height="1000" />
      </div>
      `
    );
    const $img = $('img').first();

    assert.equal(scoreByDimensions($img), 0);
  });

  it('penalizes images with small areas', () => {
    const $ = cheerio.load(
      `
      <div>
        <img src="/etc/foo.png" width="60" height="60" />
      </div>
      `
    );
    const $img = $('img').first();

    assert.equal(scoreByDimensions($img), -100);
  });

  it('prefers the largest images', () => {
    const $ = cheerio.load(
      `
      <div>
        <img src="/etc/foo.png" width="1000" height="1000" />
      </div>
      `
    );
    const $img = $('img').first();

    assert.equal(scoreByDimensions($img), 1000);
  });
});

describe('scoreByPosition($imgs, index)', () => {
  it('gives higher scores to images that come first', () => {
    const $ = cheerio.load(
      `
      <div>
        <img width="10" />
        <img width="10" />
        <img width="10" />
        <img width="10" />
        <img width="10" />
        <img width="10" />
      </div>
      `
    );
    const $imgs = $('img');

    assert.equal(scoreByPosition($imgs, 0), 3);
  });
});


<|start_filename|>src/extractors/generic/title/extractor.test.js<|end_filename|>
import assert from 'assert';
import cheerio from 'cheerio';

import HTML from './fixtures/html';
import GenericTitleExtractor from './extractor';

describe('GenericTitleExtractor', () => {
  describe('extract({ $, url, cachedMeta })', () => {
    it('extracts strong meta title tags', () => {
      const $ = cheerio.load(HTML.dcTitle.test);
      const result = GenericTitleExtractor.extract({
        $,
        url: '',
        metaCache: ['dc.title', 'something-else'],
      });

      assert.equal(result, HTML.dcTitle.result);
    });

    it('pulls title from selectors lacking string meta', () => {
      const $ = cheerio.load(HTML.strongTitleSelector.test);
      const result = GenericTitleExtractor.extract({
        $,
        url: '',
        metaCache: ['og:title', 'something-else'],
      });

      assert.equal(result, HTML.ogTitle.result);
    });

    it('then falls back to weak meta title tags', () => {
      const $ = cheerio.load(HTML.ogTitle.test);
      const result = GenericTitleExtractor.extract({
        $,
        url: '',
        metaCache: ['og:title', 'something-else'],
      });

      assert.equal(result, HTML.ogTitle.result);
    });
  });

  it('then falls back to weak selectors', () => {
    const $ = cheerio.load(HTML.weakTitleSelector.test);
    const result = GenericTitleExtractor.extract({ $, url: '', metaCache: [] });

    assert.equal(result, HTML.weakTitleSelector.result);
  });
});


<|start_filename|>src/extractors/generic/next-page-url/scoring/constants.js<|end_filename|>
export const DIGIT_RE = /\d/;

// A list of words that, if found in link text or URLs, likely mean that
// this link is not a next page link.
export const EXTRANEOUS_LINK_HINTS = [
  'print',
  'archive',
  'comment',
  'discuss',
  'e-mail',
  'email',
  'share',
  'reply',
  'all',
  'login',
  'sign',
  'single',
  'adx',
  'entry-unrelated',
];
export const EXTRANEOUS_LINK_HINTS_RE = new RegExp(
  EXTRANEOUS_LINK_HINTS.join('|'),
  'i'
);

// Match any link text/classname/id that looks like it could mean the next
// page. Things like: next, continue, >, >>, » but not >|, »| as those can
// mean last page.
export const NEXT_LINK_TEXT_RE = new RegExp(
  '(next|weiter|continue|>([^|]|$)|»([^|]|$))',
  'i'
);

// Match any link text/classname/id that looks like it is an end link: things
// like "first", "last", "end", etc.
export const CAP_LINK_TEXT_RE = new RegExp('(first|last|end)', 'i');

// Match any link text/classname/id that looks like it means the previous
// page.
export const PREV_LINK_TEXT_RE = new RegExp('(prev|earl|old|new|<|«)', 'i');

// Match any phrase that looks like it could be page, or paging, or pagination
export const PAGE_RE = new RegExp('pag(e|ing|inat)', 'i');


<|start_filename|>src/extractors/generic/date-published/fixtures/html.js<|end_filename|>
const HTML = {
  datePublishedMeta: {
    test: `
      <html>
        <head>
          <meta name="displaydate" value="1/1/2020 8:30 (EST)" />
        </head>
      </html>
    `,
    result: new Date('1/1/2020 8:30 (EST)'),
  },
  datePublishedSelectors: {
    test: `
      <div>
        <div class="hentry">
          <div class="updated">
            1/1/2020 <span class="time">8:30am</span>
          </div>
        </head>
      </div>
    `,
    result: new Date('1/1/2020 8:30 am (EST)'),
  },
};

export default HTML;


<|start_filename|>src/cleaners/author.js<|end_filename|>
import { normalizeSpaces } from 'utils/text';
import { CLEAN_AUTHOR_RE } from './constants';

// Take an author string (like 'By <NAME> ') and clean it to
// just the name(s): '<NAME>'.
export default function cleanAuthor(author) {
  return normalizeSpaces(author.replace(CLEAN_AUTHOR_RE, '$2').trim());
}


<|start_filename|>src/extractors/generic/content/scoring/get-weight.test.js<|end_filename|>
import assert from 'assert';
import cheerio from 'cheerio';

import HTML from './fixtures/get-weight';
import { getWeight } from './index';

describe('Generic Extractor Utils', () => {
  describe('getWeight(node)', () => {
    it('returns a score of 25 if node has positive id', () => {
      const $ = cheerio.load(HTML.positiveId);
      assert.equal(getWeight($('div')), 25);
    });

    it('returns a score of -25 if node has negative id', () => {
      const $ = cheerio.load(HTML.negativeId);
      assert.equal(getWeight($('div')), -25);
    });

    it('returns a score of 25 if node has positive class', () => {
      const $ = cheerio.load(HTML.positiveClass);
      assert.equal(getWeight($('div')), 25);
    });

    it('returns a score of -25 if node has negative class', () => {
      const $ = cheerio.load(HTML.negativeClass);
      assert.equal(getWeight($('div')), -25);
    });

    it('returns a score of 25 if node has both positive id and class', () => {
      const $ = cheerio.load(HTML.positiveIdAndClass);
      assert.equal(getWeight($('div')), 25);
    });

    it('returns a score of 25 if node has pos id and neg class', () => {
      // is this really wanted? id="entry" class="adbox"
      // should get positive score?
      const $ = cheerio.load(HTML.positiveIdNegClass);
      assert.equal(getWeight($('div')), 25);
    });

    it('returns a score of 10 if node has pos img class', () => {
      const $ = cheerio.load(HTML.positivePhotoClass);
      assert.equal(getWeight($('div')), 10);
    });

    it('returns a score of 35 if node has pos id pos img class', () => {
      const $ = cheerio.load(HTML.positiveIdAndPhoto);
      assert.equal(getWeight($('div')), 35);
    });

    it("adds an add'l 25 (total 50) if node uses entry-content-asset class", () => {
      const $ = cheerio.load(HTML.entryContentAsset);
      assert.equal(getWeight($('div')), 50);
    });
  });
});


<|start_filename|>src/utils/dom/get-attrs.test.js<|end_filename|>
import assert from 'assert';

import getAttrs from './get-attrs';

describe('getAttrs(node)', () => {
  it('returns attrs for a raw jquery node', () => {
    const domNode = {
      attributes: {
        0: {
          name: 'class',
          value: 'foo bar',
        },
      },
    };

    const attrs = {
      class: 'foo bar',
    };

    assert.deepEqual(getAttrs(domNode), attrs);
  });

  it('returns attrs for a raw cheerio node', () => {
    const cheerioNode = {
      attribs: {
        class: 'foo bar',
        id: 'baz bat',
      },
    };

    assert.deepEqual(getAttrs(cheerioNode), cheerioNode.attribs);
  });
});


<|start_filename|>src/extractors/custom/money.cnn.com/index.js<|end_filename|>
export const MoneyCnnComExtractor = {
  domain: 'money.cnn.com',

  title: {
    selectors: ['.article-title'],
  },

  author: {
    selectors: ['.byline a'],
  },

  date_published: {
    selectors: [['meta[name="date"]', 'value']],

    timezone: 'GMT',
  },

  dek: {
    selectors: ['#storytext h2'],
  },

  lead_image_url: {
    selectors: [['meta[name="og:image"]', 'value']],
  },

  content: {
    selectors: ['#storytext'],

    // Is there anything in the content you selected that needs transformed
    // before it's consumable content? E.g., unusual lazy loaded images
    transforms: {},

    // Is there anything that is in the result that shouldn't be?
    // The clean selectors will remove anything that matches from
    // the result
    clean: ['.inStoryHeading'],
  },
};


<|start_filename|>src/extractors/generic/lead-image-url/extractor.js<|end_filename|>
import { extractFromMeta } from 'utils/dom';
import { cleanImage } from 'cleaners';

import {
  LEAD_IMAGE_URL_META_TAGS,
  LEAD_IMAGE_URL_SELECTORS,
} from './constants';

import {
  scoreImageUrl,
  scoreAttr,
  scoreByParents,
  scoreBySibling,
  scoreByDimensions,
  scoreByPosition,
} from './score-image';

// Given a resource, try to find the lead image URL from within
// it. Like content and next page extraction, uses a scoring system
// to determine what the most likely image may be. Short circuits
// on really probable things like og:image meta tags.
//
// Potential signals to still take advantage of:
//   * domain
//   * weird aspect ratio
const GenericLeadImageUrlExtractor = {
  extract({ $, content, metaCache, html }) {
    let cleanUrl;
    if (!$.browser && $('head').length === 0) {
      $('*')
        .first()
        .prepend(html);
    }

    // Check to see if we have a matching meta tag that we can make use of.
    // Moving this higher because common practice is now to use large
    // images on things like Open Graph or Twitter cards.
    // images usually have for things like Open Graph.
    const imageUrl = extractFromMeta(
      $,
      LEAD_IMAGE_URL_META_TAGS,
      metaCache,
      false
    );

    if (imageUrl) {
      cleanUrl = cleanImage(imageUrl);

      if (cleanUrl) return cleanUrl;
    }

    // Next, try to find the "best" image via the content.
    // We'd rather not have to fetch each image and check dimensions,
    // so try to do some analysis and determine them instead.
    const $content = $(content);
    const imgs = $('img', $content).toArray();
    const imgScores = {};

    imgs.forEach((img, index) => {
      const $img = $(img);
      const src = $img.attr('src');

      if (!src) return;

      let score = scoreImageUrl(src);
      score += scoreAttr($img);
      score += scoreByParents($img);
      score += scoreBySibling($img);
      score += scoreByDimensions($img);
      score += scoreByPosition(imgs, index);

      imgScores[src] = score;
    });

    const [topUrl, topScore] = Reflect.ownKeys(imgScores).reduce(
      (acc, key) => (imgScores[key] > acc[1] ? [key, imgScores[key]] : acc),
      [null, 0]
    );

    if (topScore > 0) {
      cleanUrl = cleanImage(topUrl);

      if (cleanUrl) return cleanUrl;
    }

    // If nothing else worked, check to see if there are any really
    // probable nodes in the doc, like <link rel="image_src" />.
    // eslint-disable-next-line no-restricted-syntax
    for (const selector of LEAD_IMAGE_URL_SELECTORS) {
      const $node = $(selector).first();
      const src = $node.attr('src');
      if (src) {
        cleanUrl = cleanImage(src);
        if (cleanUrl) return cleanUrl;
      }

      const href = $node.attr('href');
      if (href) {
        cleanUrl = cleanImage(href);
        if (cleanUrl) return cleanUrl;
      }

      const value = $node.attr('value');
      if (value) {
        cleanUrl = cleanImage(value);
        if (cleanUrl) return cleanUrl;
      }
    }

    return null;
  },
};

export default GenericLeadImageUrlExtractor;


<|start_filename|>src/utils/dom/is-wordpress.js<|end_filename|>
import { IS_WP_SELECTOR } from './constants';

export default function isWordpress($) {
  return $(IS_WP_SELECTOR).length > 0;
}


<|start_filename|>src/cleaners/content.js<|end_filename|>
import {
  cleanAttributes,
  cleanHeaders,
  cleanHOnes,
  cleanImages,
  cleanTags,
  removeEmpty,
  rewriteTopLevel,
  markToKeep,
  stripJunkTags,
  makeLinksAbsolute,
} from 'utils/dom';

// Clean our article content, returning a new, cleaned node.
export default function extractCleanNode(
  article,
  { $, cleanConditionally = true, title = '', url = '', defaultCleaner = true }
) {
  // Rewrite the tag name to div if it's a top level node like body or
  // html to avoid later complications with multiple body tags.
  rewriteTopLevel(article, $);

  // Drop small images and spacer images
  // Only do this is defaultCleaner is set to true;
  // this can sometimes be too aggressive.
  if (defaultCleaner) cleanImages(article, $);

  // Make links absolute
  makeLinksAbsolute(article, $, url);

  // Mark elements to keep that would normally be removed.
  // E.g., stripJunkTags will remove iframes, so we're going to mark
  // YouTube/Vimeo videos as elements we want to keep.
  markToKeep(article, $, url);

  // Drop certain tags like <title>, etc
  // This is -mostly- for cleanliness, not security.
  stripJunkTags(article, $);

  // H1 tags are typically the article title, which should be extracted
  // by the title extractor instead. If there's less than 3 of them (<3),
  // strip them. Otherwise, turn 'em into H2s.
  cleanHOnes(article, $);

  // Clean headers
  cleanHeaders(article, $, title);

  // We used to clean UL's and OL's here, but it was leading to
  // too many in-article lists being removed. Consider a better
  // way to detect menus particularly and remove them.
  // Also optionally running, since it can be overly aggressive.
  if (defaultCleaner) cleanTags(article, $, cleanConditionally);

  // Remove empty paragraph nodes
  removeEmpty(article, $);

  // Remove unnecessary attributes
  cleanAttributes(article, $);

  return article;
}


<|start_filename|>src/utils/text/index.js<|end_filename|>
export { default as normalizeSpaces } from './normalize-spaces';
export { default as extractFromUrl } from './extract-from-url';
export { default as pageNumFromUrl } from './page-num-from-url';
export { default as removeAnchor } from './remove-anchor';
export { default as articleBaseUrl } from './article-base-url';
export { default as hasSentenceEnd } from './has-sentence-end';
export { default as excerptContent } from './excerpt-content';
export { default as getEncoding } from './get-encoding';


<|start_filename|>src/extractors/custom/blisterreview.com/index.js<|end_filename|>
export const BlisterreviewComExtractor = {
  domain: 'blisterreview.com',

  title: {
    selectors: [['meta[name="og:title"]', 'value'], 'h1.entry-title'],
  },

  author: {
    selectors: ['span.author-name'],
  },

  date_published: {
    selectors: [
      ['meta[name="article:published_time"]', 'value'],
      ['time.entry-date', 'datetime'],
      ['meta[itemprop="datePublished"]', 'content'],
    ],
  },

  dek: {
    selectors: [
      // enter selectors
    ],
  },

  lead_image_url: {
    selectors: [
      ['meta[name="og:image"]', 'value'],
      ['meta[property="og:image"]', 'content'],
      ['meta[itemprop="image"]', 'content'],
      ['meta[name="twitter:image"]', 'content'],
      ['img.attachment-large', 'src'],
    ],
  },

  content: {
    selectors: [
      [
        '.elementor-section-wrap',
        '.elementor-text-editor > p, .elementor-text-editor > ul > li, .attachment-large, .wp-caption-text',
      ],
    ],

    // Is there anything in the content you selected that needs transformed
    // before it's consumable content? E.g., unusual lazy loaded images
    transforms: {
      figcaption: 'p',
    },

    // Is there anything that is in the result that shouldn't be?
    // The clean selectors will remove anything that matches from
    // the result
    clean: ['.comments-area'],
  },
};


<|start_filename|>src/extractors/generic/content/scoring/get-weight.js<|end_filename|>
import {
  NEGATIVE_SCORE_RE,
  POSITIVE_SCORE_RE,
  PHOTO_HINTS_RE,
  READABILITY_ASSET,
} from './constants';

// Get the score of a node based on its className and id.
export default function getWeight(node) {
  const classes = node.attr('class');
  const id = node.attr('id');
  let score = 0;

  if (id) {
    // if id exists, try to score on both positive and negative
    if (POSITIVE_SCORE_RE.test(id)) {
      score += 25;
    }
    if (NEGATIVE_SCORE_RE.test(id)) {
      score -= 25;
    }
  }

  if (classes) {
    if (score === 0) {
      // if classes exist and id did not contribute to score
      // try to score on both positive and negative
      if (POSITIVE_SCORE_RE.test(classes)) {
        score += 25;
      }
      if (NEGATIVE_SCORE_RE.test(classes)) {
        score -= 25;
      }
    }

    // even if score has been set by id, add score for
    // possible photo matches
    // "try to keep photos if we can"
    if (PHOTO_HINTS_RE.test(classes)) {
      score += 10;
    }

    // add 25 if class matches entry-content-asset,
    // a class apparently instructed for use in the
    // Readability publisher guidelines
    // https://www.readability.com/developers/guidelines
    if (READABILITY_ASSET.test(classes)) {
      score += 25;
    }
  }

  return score;
}


<|start_filename|>src/utils/dom/node-is-sufficient.js<|end_filename|>
// Given a node, determine if it's article-like enough to return
// param: node (a cheerio node)
// return: boolean

export default function nodeIsSufficient($node) {
  return $node.text().trim().length >= 100;
}


<|start_filename|>src/utils/text/has-sentence-end.js<|end_filename|>
// Given a string, return True if it appears to have an ending sentence
// within it, false otherwise.
const SENTENCE_END_RE = new RegExp('.( |$)');
export default function hasSentenceEnd(text) {
  return SENTENCE_END_RE.test(text);
}


<|start_filename|>src/cleaners/resolve-split-title.js<|end_filename|>
import URL from 'url';
import wuzzy from 'wuzzy';

import { TITLE_SPLITTERS_RE, DOMAIN_ENDINGS_RE } from './constants';

function extractBreadcrumbTitle(splitTitle, text) {
  // This must be a very breadcrumbed title, like:
  // The Best Gadgets on Earth : Bits : Blogs : NYTimes.com
  // NYTimes - Blogs - Bits - The Best Gadgets on Earth
  if (splitTitle.length >= 6) {
    // Look to see if we can find a breadcrumb splitter that happens
    // more than once. If we can, we'll be able to better pull out
    // the title.
    const termCounts = splitTitle.reduce((acc, titleText) => {
      acc[titleText] = acc[titleText] ? acc[titleText] + 1 : 1;
      return acc;
    }, {});

    const [maxTerm, termCount] = Reflect.ownKeys(termCounts).reduce(
      (acc, key) => {
        if (acc[1] < termCounts[key]) {
          return [key, termCounts[key]];
        }

        return acc;
      },
      [0, 0]
    );

    // We found a splitter that was used more than once, so it
    // is probably the breadcrumber. Split our title on that instead.
    // Note: max_term should be <= 4 characters, so that " >> "
    // will match, but nothing longer than that.
    if (termCount >= 2 && maxTerm.length <= 4) {
      splitTitle = text.split(maxTerm);
    }

    const splitEnds = [splitTitle[0], splitTitle.slice(-1)];
    const longestEnd = splitEnds.reduce(
      (acc, end) => (acc.length > end.length ? acc : end),
      ''
    );

    if (longestEnd.length > 10) {
      return longestEnd;
    }

    return text;
  }

  return null;
}

function cleanDomainFromTitle(splitTitle, url) {
  // Search the ends of the title, looking for bits that fuzzy match
  // the URL too closely. If one is found, discard it and return the
  // rest.
  //
  // Strip out the big TLDs - it just makes the matching a bit more
  // accurate. Not the end of the world if it doesn't strip right.
  const { host } = URL.parse(url);
  const nakedDomain = host.replace(DOMAIN_ENDINGS_RE, '');

  const startSlug = splitTitle[0].toLowerCase().replace(' ', '');
  const startSlugRatio = wuzzy.levenshtein(startSlug, nakedDomain);

  if (startSlugRatio > 0.4 && startSlug.length > 5) {
    return splitTitle.slice(2).join('');
  }

  const endSlug = splitTitle
    .slice(-1)[0]
    .toLowerCase()
    .replace(' ', '');
  const endSlugRatio = wuzzy.levenshtein(endSlug, nakedDomain);

  if (endSlugRatio > 0.4 && endSlug.length >= 5) {
    return splitTitle.slice(0, -2).join('');
  }

  return null;
}

// Given a title with separators in it (colons, dashes, etc),
// resolve whether any of the segments should be removed.
export default function resolveSplitTitle(title, url = '') {
  // Splits while preserving splitters, like:
  // ['The New New York', ' - ', 'The Washington Post']
  const splitTitle = title.split(TITLE_SPLITTERS_RE);
  if (splitTitle.length === 1) {
    return title;
  }

  let newTitle = extractBreadcrumbTitle(splitTitle, title);
  if (newTitle) return newTitle;

  newTitle = cleanDomainFromTitle(splitTitle, url);
  if (newTitle) return newTitle;

  // Fuzzy ratio didn't find anything, so this title is probably legit.
  // Just return it all.
  return title;
}


<|start_filename|>src/extractors/custom/pagesix.com/index.js<|end_filename|>
export const PagesixComExtractor = {
  domain: 'pagesix.com',

  supportedDomains: ['nypost.com'],

  title: {
    selectors: ['h1 a'],
  },

  author: {
    selectors: ['.byline'],
  },

  date_published: {
    selectors: [['meta[name="article:published_time"]', 'value']],
  },

  dek: {
    selectors: [['meta[name="description"]', 'value']],
  },

  lead_image_url: {
    selectors: [['meta[name="og:image"]', 'value']],
  },

  content: {
    selectors: [
      ['#featured-image-wrapper', '.entry-content'],
      '.entry-content',
    ],

    // Is there anything in the content you selected that needs transformed
    // before it's consumable content? E.g., unusual lazy loaded images
    transforms: {
      '#featured-image-wrapper': 'figure',
      '.wp-caption-text': 'figcaption',
    },

    // Is there anything that is in the result that shouldn't be?
    // The clean selectors will remove anything that matches from
    // the result
    clean: ['.modal-trigger'],
  },
};


<|start_filename|>src/extractors/index.js<|end_filename|>
const Extractor = {};

export default Extractor;


<|start_filename|>src/extractors/custom/www.fastcompany.com/index.js<|end_filename|>
export const WwwFastcompanyComExtractor = {
  domain: 'www.fastcompany.com',

  title: {
    selectors: ['h1'],
  },

  author: {
    selectors: ['.post__by'],
  },

  date_published: {
    selectors: [['meta[name="article:published_time"]', 'value']],
  },

  dek: {
    selectors: ['.post__deck'],
  },

  lead_image_url: {
    selectors: [['meta[name="og:image"]', 'value']],
  },

  content: {
    selectors: ['.post__article'],
  },
};


<|start_filename|>src/extractors/generic/content/scoring/index.js<|end_filename|>
// Scoring
export { default as getWeight } from './get-weight';
export { default as getScore } from './get-score';
export { default as scoreCommas } from './score-commas';
export { default as scoreLength } from './score-length';
export { default as scoreParagraph } from './score-paragraph';
export { default as setScore } from './set-score';
export { default as addScore } from './add-score';
export { default as addToParent } from './add-to-parent';
export { default as getOrInitScore } from './get-or-init-score';
export { default as scoreNode } from './score-node';
export { default as scoreContent } from './score-content';
export { default as findTopCandidate } from './find-top-candidate';


<|start_filename|>src/extractors/custom/nymag.com/index.test.js<|end_filename|>
import assert from 'assert';

import Mercury from 'mercury';

const fs = require('fs');

describe('NYMagExtractor', () => {
  it('works with a feature story', async () => {
    const html = fs.readFileSync('./fixtures/nymag.com/ailes.html');
    const uri =
      'http://nymag.com/daily/intelligencer/2016/09/how-fox-news-women-took-down-roger-ailes.html';

    const { dek, title, author } = await Mercury.parse(uri, html);
    const actualDek =
      'How Fox News women took down the most powerful, and predatory, man in media.';

    assert.equal(dek, actualDek);
    assert.equal(title, 'The Revenge of Roger’s Angels');
    assert.equal(author, '<NAME>');
  });
});


<|start_filename|>src/extractors/custom/wikipedia.org/index.js<|end_filename|>
export const WikipediaExtractor = {
  domain: 'wikipedia.org',
  content: {
    selectors: ['#mw-content-text'],

    defaultCleaner: false,

    // transform top infobox to an image with caption
    transforms: {
      '.infobox img': $node => {
        const $parent = $node.parents('.infobox');
        // Only prepend the first image in .infobox
        if ($parent.children('img').length === 0) {
          $parent.prepend($node);
        }
      },
      '.infobox caption': 'figcaption',
      '.infobox': 'figure',
    },

    // Selectors to remove from the extracted content
    clean: [
      '.mw-editsection',
      'figure tr, figure td, figure tbody',
      '#toc',
      '.navbox',
    ],
  },

  author: 'Wikipedia Contributors',

  title: {
    selectors: ['h2.title'],
  },

  date_published: {
    selectors: ['#footer-info-lastmod'],
  },
};


<|start_filename|>src/extractors/constants.js<|end_filename|>
export const ATTR_RE = /\[([\w-]+)\]/; // eslint-disable-line no-useless-escape


<|start_filename|>src/resource/utils/dom/clean.js<|end_filename|>
import { TAGS_TO_REMOVE } from './constants';

function isComment(index, node) {
  return node.type === 'comment';
}

function cleanComments($) {
  $.root()
    .find('*')
    .contents()
    .filter(isComment)
    .remove();

  return $;
}

export default function clean($) {
  $(TAGS_TO_REMOVE).remove();

  $ = cleanComments($);
  return $;
}


<|start_filename|>src/utils/validate-url.js<|end_filename|>
// extremely simple url validation as a first step
export default function validateUrl({ hostname }) {
  // If this isn't a valid url, return an error message
  return !!hostname;
}


<|start_filename|>scripts/generate-custom-parser.js<|end_filename|>
/* eslint-disable import/no-extraneous-dependencies */
/* eslint-disable no-use-before-define */
/* eslint-disable no-console */
import fs from 'fs';
import URL from 'url';
import inquirer from 'inquirer';
import ora from 'ora';
import { exec } from 'child_process';

import { stripJunkTags, makeLinksAbsolute } from 'utils/dom';
import Mercury from '../dist/mercury';
import extractorTemplate from './templates/custom-extractor';
import extractorTestTemplate from './templates/custom-extractor-test';

const questions = [
  {
    type: 'input',
    name: 'website',
    message:
      "Paste a url to an article you'd like to create or extend a parser for:",
    validate(value) {
      const { hostname } = URL.parse(value);
      if (hostname) return true;

      return false;
    },
  },
];
let spinner;

function confirm(fn, args, msg, newParser) {
  spinner = ora({ text: msg });
  spinner.start();
  const result = fn(...args);

  if (result && result.then) {
    result.then(r => savePage(r, args, newParser));
  } else {
    spinner.succeed();
  }

  return result;
}

function confirmCreateDir(dir, msg) {
  if (!fs.existsSync(dir)) {
    confirm(fs.mkdirSync, [dir], msg);
  }
}

function getDir(url) {
  const { hostname } = URL.parse(url);
  return `./src/extractors/custom/${hostname}`;
}

function scaffoldCustomParser(url) {
  const dir = getDir(url);
  const { hostname } = URL.parse(url);
  let newParser = false;

  if (!fs.existsSync(dir)) {
    newParser = true;
    confirmCreateDir(dir, `Creating ${hostname} directory`);
    confirmCreateDir(`./fixtures/${hostname}`, 'Creating fixtures directory');
  }

  confirm(Mercury.fetchResource, [url], 'Fetching fixture', newParser);
}

// if has arg, just assume that arg is a url and skip prmopt
const urlArg = process.argv[2];
if (urlArg) {
  scaffoldCustomParser(urlArg);
} else {
  inquirer.prompt(questions).then(answers => {
    scaffoldCustomParser(answers.website);
  });
}

function generateScaffold(url, file, result) {
  const { hostname } = URL.parse(url);
  const extractor = extractorTemplate(hostname, extractorName(hostname));
  const extractorTest = extractorTestTemplate(
    file,
    url,
    getDir(url),
    result,
    extractorName(hostname)
  );

  fs.writeFileSync(`${getDir(url)}/index.js`, extractor);
  fs.writeFileSync(`${getDir(url)}/index.test.js`, extractorTest);
  fs.appendFileSync('./src/extractors/custom/index.js', exportString(url));
  exec(`npm run lint-fix-quiet -- ${getDir(url)}/*.js`);
}

function savePage($, [url], newParser) {
  const { hostname } = URL.parse(url);

  spinner.succeed();

  const filename = new Date().getTime();
  const file = `./fixtures/${hostname}/${filename}.html`;
  // fix http(s) relative links:
  makeLinksAbsolute($('*').first(), $, url);
  $('[src], [href]').each((index, node) => {
    const $node = $(node);
    const link = $node.attr('src');
    if (link && link.slice(0, 2) === '//') {
      $node.attr('src', `http:${link}`);
    }
  });
  const html = stripJunkTags($('*').first(), $, ['script']).html();

  fs.writeFileSync(file, html);

  Mercury.parse(url, { html }).then(result => {
    if (newParser) {
      confirm(
        generateScaffold,
        [url, file, result],
        'Generating parser and tests'
      );
      console.log(`Your custom site extractor has been set up. To get started building it, run
      yarn watch:test -- ${hostname}
        -- OR --
      npm run watch:test -- ${hostname}`);
    } else {
      console.log(`
  It looks like you already have a custom parser for this url.
  The page you linked to has been added to ${file}. Copy and paste
  the following code to use that page in your tests:
  const html = fs.readFileSync('${file}');`);
    }
  });
}

function exportString(url) {
  const { hostname } = URL.parse(url);
  return `export * from './${hostname}';`;
}

function extractorName(hostname) {
  const name = hostname
    .split('.')
    .map(w => `${w.charAt(0).toUpperCase()}${w.slice(1)}`)
    .join('');
  return `${name}Extractor`;
}


<|start_filename|>src/utils/dom/clean-images.js<|end_filename|>
import { SPACER_RE } from './constants';

function cleanForHeight($img, $) {
  const height = parseInt($img.attr('height'), 10);
  const width = parseInt($img.attr('width'), 10) || 20;

  // Remove images that explicitly have very small heights or
  // widths, because they are most likely shims or icons,
  // which aren't very useful for reading.
  if ((height || 20) < 10 || width < 10) {
    $img.remove();
  } else if (height) {
    // Don't ever specify a height on images, so that we can
    // scale with respect to width without screwing up the
    // aspect ratio.
    $img.removeAttr('height');
  }

  return $;
}

// Cleans out images where the source string matches transparent/spacer/etc
// TODO This seems very aggressive - AP
function removeSpacers($img, $) {
  if (SPACER_RE.test($img.attr('src'))) {
    $img.remove();
  }

  return $;
}

export default function cleanImages($article, $) {
  $article.find('img').each((index, img) => {
    const $img = $(img);

    cleanForHeight($img, $);
    removeSpacers($img, $);
  });

  return $;
}


<|start_filename|>src/extractors/generic/next-page-url/scoring/score-links.js<|end_filename|>
import URL from 'url';

import { getAttrs, isWordpress } from 'utils/dom';
import { removeAnchor, pageNumFromUrl } from 'utils/text';

import {
  scoreSimilarity,
  scoreLinkText,
  scorePageInLink,
  scoreExtraneousLinks,
  scoreByParents,
  scorePrevLink,
  shouldScore,
  scoreBaseUrl,
  scoreCapLinks,
  scoreNextLinkText,
} from './utils';

export function makeBaseRegex(baseUrl) {
  return new RegExp(`^${baseUrl}`, 'i');
}

function makeSig($link, linkText) {
  return `${linkText || $link.text()} ${$link.attr('class') || ''} ${$link.attr(
    'id'
  ) || ''}`;
}

export default function scoreLinks({
  links,
  articleUrl,
  baseUrl,
  parsedUrl,
  $,
  previousUrls = [],
}) {
  parsedUrl = parsedUrl || URL.parse(articleUrl);
  const baseRegex = makeBaseRegex(baseUrl);
  const isWp = isWordpress($);

  // Loop through all links, looking for hints that they may be next-page
  // links. Things like having "page" in their textContent, className or
  // id, or being a child of a node with a page-y className or id.
  //
  // After we do that, assign each page a score, and pick the one that
  // looks most like the next page link, as long as its score is strong
  // enough to have decent confidence.
  const scoredPages = links.reduce((possiblePages, link) => {
    // Remove any anchor data since we don't do a good job
    // standardizing URLs (it's hard), we're going to do
    // some checking with and without a trailing slash
    const attrs = getAttrs(link);

    // if href is undefined, return
    if (!attrs.href) return possiblePages;

    const href = removeAnchor(attrs.href);
    const $link = $(link);
    const linkText = $link.text();

    if (
      !shouldScore(href, articleUrl, baseUrl, parsedUrl, linkText, previousUrls)
    ) {
      return possiblePages;
    }

    // ## PASSED THE FIRST-PASS TESTS. Start scoring. ##
    if (!possiblePages[href]) {
      possiblePages[href] = {
        score: 0,
        linkText,
        href,
      };
    } else {
      possiblePages[href].linkText = `${
        possiblePages[href].linkText
      }|${linkText}`;
    }

    const possiblePage = possiblePages[href];
    const linkData = makeSig($link, linkText);
    const pageNum = pageNumFromUrl(href);

    let score = scoreBaseUrl(href, baseRegex);
    score += scoreNextLinkText(linkData);
    score += scoreCapLinks(linkData);
    score += scorePrevLink(linkData);
    score += scoreByParents($link);
    score += scoreExtraneousLinks(href);
    score += scorePageInLink(pageNum, isWp);
    score += scoreLinkText(linkText, pageNum);
    score += scoreSimilarity(score, articleUrl, href);

    possiblePage.score = score;

    return possiblePages;
  }, {});

  return Reflect.ownKeys(scoredPages).length === 0 ? null : scoredPages;
}


<|start_filename|>src/extractors/generic/content/scoring/get-score.test.js<|end_filename|>
import assert from 'assert';
import cheerio from 'cheerio';

import { getScore } from './index';

describe('Scoring utils', () => {
  describe('getScore($node)', () => {
    it('returns null if the node has no score set', () => {
      const $ = cheerio.load('<p>Foo</p>');
      const $node = $('p').first();
      assert.equal(getScore($node), null);
    });

    it('returns 25 if the node has a score attr of 25', () => {
      const $ = cheerio.load('<p score="25">Foo</p>');
      const $node = $('p').first();
      assert.equal(typeof getScore($node), 'number');
      assert.equal(getScore($node), 25);
    });
  });
});


<|start_filename|>src/extractors/generic/next-page-url/scoring/utils/score-cap-links.test.js<|end_filename|>
import assert from 'assert';

import scoreCapLinks from './score-cap-links';

describe('scoreCapLinks(linkData)', () => {
  it('returns -65 if cap link with next link text', () => {
    const linkData = 'foo next Last page';

    assert.equal(scoreCapLinks(linkData), -65);
  });

  it('returns 0 if does not match a cap link', () => {
    const linkData = 'foo bar WOW GREAT';

    assert.equal(scoreCapLinks(linkData), 0);
  });
});


<|start_filename|>src/utils/dom/strip-junk-tags.js<|end_filename|>
import { STRIP_OUTPUT_TAGS, KEEP_CLASS } from './constants';

export default function stripJunkTags(article, $, tags = []) {
  if (tags.length === 0) {
    tags = STRIP_OUTPUT_TAGS;
  }

  // Remove matching elements, but ignore
  // any element with a class of mercury-parser-keep
  $(tags.join(','), article)
    .not(`.${KEEP_CLASS}`)
    .remove();

  return $;
}


<|start_filename|>src/extractors/generic/next-page-url/scoring/utils/score-link-text.js<|end_filename|>
import { IS_DIGIT_RE } from 'utils/text/constants';

export default function scoreLinkText(linkText, pageNum) {
  // If the link text can be parsed as a number, give it a minor
  // bonus, with a slight bias towards lower numbered pages. This is
  // so that pages that might not have 'next' in their text can still
  // get scored, and sorted properly by score.
  let score = 0;

  if (IS_DIGIT_RE.test(linkText.trim())) {
    const linkTextAsNum = parseInt(linkText, 10);
    // If it's the first page, we already got it on the first call.
    // Give it a negative score. Otherwise, up to page 10, give a
    // small bonus.
    if (linkTextAsNum < 2) {
      score = -30;
    } else {
      score = Math.max(0, 10 - linkTextAsNum);
    }

    // If it appears that the current page number is greater than
    // this links page number, it's a very bad sign. Give it a big
    // penalty.
    if (pageNum && pageNum >= linkTextAsNum) {
      score -= 50;
    }
  }

  return score;
}


<|start_filename|>scripts/generate-fixture-preview.js<|end_filename|>
/* eslint-disable */
const page = require('webpage').create();
const system = require('system');
const args = system.args;

const fixtures = args
  .slice(1)[0]
  .slice(0, -1)
  .split(',');
const totalRenders = fixtures.length;

var renderCount = 0;
function pageRenderComplete() {
  renderCount++;
  if (renderCount === totalRenders) {
    phantom.exit();
  } else {
    capturePage();
  }
}

function capturePage() {
  const fixturePath = fixtures[renderCount];
  page.viewportSize = { width: 1366, height: 768 };

  page.open(fixturePath, function() {
    // set default background to white (otherwise can sometimes get transparent bg in png
    const script =
      "function() { \
      var style = document.createElement('style'); \
      var text = document.createTextNode('body { background: #fff }'); \
      style.setAttribute('type', 'text/css'); \
      style.appendChild(text); \
      document.head.insertBefore(style, document.head.firstChild); \
    }";

    page.evaluateJavaScript(script);

    var filename = new Date();
    page.clipRect = { top: 0, left: 0, width: 1366, height: 768 };
    page.render('tmp/artifacts/' + fixturePath + '.png');
    pageRenderComplete();
  });
}

capturePage();


<|start_filename|>src/extractors/generic/next-page-url/scoring/utils/score-by-parents.js<|end_filename|>
import { range } from 'utils';
import {
  NEGATIVE_SCORE_RE,
  POSITIVE_SCORE_RE,
  PAGE_RE,
} from 'utils/dom/constants';
import { EXTRANEOUS_LINK_HINTS_RE } from '../constants';

function makeSig($link) {
  return `${$link.attr('class') || ''} ${$link.attr('id') || ''}`;
}

export default function scoreByParents($link) {
  // If a parent node contains paging-like classname or id, give a
  // bonus. Additionally, if a parent_node contains bad content
  // (like 'sponsor'), give a penalty.
  let $parent = $link.parent();
  let positiveMatch = false;
  let negativeMatch = false;
  let score = 0;

  Array.from(range(0, 4)).forEach(() => {
    if ($parent.length === 0) {
      return;
    }

    const parentData = makeSig($parent, ' ');

    // If we have 'page' or 'paging' in our data, that's a good
    // sign. Add a bonus.
    if (!positiveMatch && PAGE_RE.test(parentData)) {
      positiveMatch = true;
      score += 25;
    }

    // If we have 'comment' or something in our data, and
    // we don't have something like 'content' as well, that's
    // a bad sign. Give a penalty.
    if (
      !negativeMatch &&
      NEGATIVE_SCORE_RE.test(parentData) &&
      EXTRANEOUS_LINK_HINTS_RE.test(parentData)
    ) {
      if (!POSITIVE_SCORE_RE.test(parentData)) {
        negativeMatch = true;
        score -= 25;
      }
    }

    $parent = $parent.parent();
  });

  return score;
}


<|start_filename|>src/resource/utils/dom/normalize-meta-tags.js<|end_filename|>
function convertMetaProp($, from, to) {
  $(`meta[${from}]`).each((_, node) => {
    const $node = $(node);

    const value = $node.attr(from);
    $node.attr(to, value);
    $node.removeAttr(from);
  });

  return $;
}

// For ease of use in extracting from meta tags,
// replace the "content" attribute on meta tags with the
// "value" attribute.
//
// In addition, normalize 'property' attributes to 'name' for ease of
// querying later. See, e.g., og or twitter meta tags.

export default function normalizeMetaTags($) {
  $ = convertMetaProp($, 'content', 'value');
  $ = convertMetaProp($, 'property', 'name');
  return $;
}


<|start_filename|>src/extractors/generic/content/scoring/score-node.test.js<|end_filename|>
import assert from 'assert';
import cheerio from 'cheerio';

import HTML from './fixtures/html';

import { scoreNode, scoreParagraph } from './index';

describe('scoreNode(node)', () => {
  it('scores P, LI, SPAN, and PRE using scoreParagraph', () => {
    const html = '<p><em>Foo</em> bar</p>';
    const $ = cheerio.load(html);
    const node = $('p').first();

    const score = scoreNode(node);
    const pScore = scoreParagraph(node);

    assert.equal(score, pScore);
    assert.equal(score, 0);
  });

  it('scores P, LI, SPAN, and PRE using scoreParagraph', () => {
    const $ = cheerio.load(HTML.score1);
    const node = $('p').first();

    const score = scoreNode(node);
    const pScore = scoreParagraph(node);

    assert.equal(score, pScore);
    assert.equal(score, 1);
  });

  it('scores P, LI, SPAN, and PRE using scoreParagraph', () => {
    const $ = cheerio.load(HTML.score3);
    const node = $('p').first();

    const score = scoreNode(node);
    const pScore = scoreParagraph(node);

    assert.equal(score, pScore);
    assert.equal(score, 3);
  });

  it('scores P, LI, SPAN, and PRE using scoreParagraph', () => {
    const $ = cheerio.load(HTML.score19);
    const node = $('p').first();

    const score = scoreNode(node);
    const pScore = scoreParagraph(node);

    assert.equal(score, pScore);
    assert.equal(score, 19);
  });

  it('scores divs with 5', () => {
    const $ = cheerio.load(HTML.divScore5);
    const node = $('div').first();

    const score = scoreNode(node);

    assert.equal(score, 5);
  });

  it('scores the blockquote family with 3', () => {
    const $ = cheerio.load(HTML.blockquoteScore3);
    const node = $('blockquote').first();

    const score = scoreNode(node);

    assert.equal(score, 3);
  });

  it('scores a form with negative 3', () => {
    const $ = cheerio.load(HTML.formScoreNeg3);
    const node = $('form').first();

    const score = scoreNode(node);

    assert.equal(score, -3);
  });

  it('scores a TH element with negative 5', () => {
    const $ = cheerio.load(HTML.thScoreNeg5);
    const node = $('th').first();

    const score = scoreNode(node);

    assert.equal(score, -5);
  });
});


<|start_filename|>src/extractors/generic/title/fixtures/html.js<|end_filename|>
const HTML = {
  dcTitle: {
    test: `
      <html>
        <meta name="dc.title" value="This Is the Title Okay" />
      <html>
    `,
    result: 'This Is the Title Okay',
  },
  ogTitle: {
    test: `
      <html>
        <meta name="og:title" value="This Is the Title Okay" />
      <html>
    `,
    result: 'This Is the Title Okay',
  },
  strongTitleSelector: {
    test: `
      <html>
        <article class="hentry">
          <h1 class="entry-title">This Is the Title Okay</h1>
        </article>
      <html>
    `,
    result: 'This Is the Title Okay',
  },
  weakTitleSelector: {
    test: `
      <html>
        <head>
          <title>This Is the Weak Title Okay</title>
        </head>
      <html>
    `,
    result: 'This Is the Weak Title Okay',
  },
};

export default HTML;


<|start_filename|>src/extractors/generic/content/extract-best-node.test.js<|end_filename|>
import assert from 'assert';
import cheerio from 'cheerio';
import extractBestNode from './extract-best-node';

const fs = require('fs');

describe('extractBestNode($, flags)', () => {
  it('scores the dom nodes and returns the best option', () => {
    const html = fs.readFileSync('./fixtures/latimes.html', 'utf-8');
    const opts = {
      stripUnlikelyCandidates: true,
      weightNodes: true,
    };

    const $ = cheerio.load(html);

    const bestNode = extractBestNode($, opts);

    assert(typeof bestNode, 'object');
  });
});


<|start_filename|>src/utils/text/constants.js<|end_filename|>
// An expression that looks to try to find the page digit within a URL, if
// it exists.
// Matches:
//  page=1
//  pg=1
//  p=1
//  paging=12
//  pag=7
//  pagination/1
//  paging/88
//  pa/83
//  p/11
//
// Does not match:
//  pg=102
//  page:2
export const PAGE_IN_HREF_RE = new RegExp(
  '(page|paging|(p(a|g|ag)?(e|enum|ewanted|ing|ination)))?(=|/)([0-9]{1,3})',
  'i'
);

export const HAS_ALPHA_RE = /[a-z]/i;

export const IS_ALPHA_RE = /^[a-z]+$/i;
export const IS_DIGIT_RE = /^[0-9]+$/i;

export const ENCODING_RE = /charset=([\w-]+)\b/;
export const DEFAULT_ENCODING = 'utf-8';


<|start_filename|>src/extractors/custom/news.nationalgeographic.com/index.js<|end_filename|>
export const NewsNationalgeographicComExtractor = {
  domain: 'news.nationalgeographic.com',

  title: {
    selectors: ['h1', 'h1.main-title'],
  },

  author: {
    selectors: ['.byline-component__contributors b span'],
  },

  date_published: {
    selectors: [['meta[name="article:published_time"]', 'value']],
    format: 'ddd MMM DD HH:mm:ss zz YYYY',
    timezone: 'EST',
  },

  dek: {
    selectors: ['.article__deck'],
  },

  lead_image_url: {
    selectors: [['meta[name="og:image"]', 'value']],
  },

  content: {
    selectors: [['.parsys.content', '.__image-lead__'], '.content'],

    // Is there anything in the content you selected that needs transformed
    // before it's consumable content? E.g., unusual lazy loaded images
    transforms: {
      '.parsys.content': ($node, $) => {
        const $imgSrc = $node
          .find('.image.parbase.section')
          .find('.picturefill')
          .first()
          .data('platform-src');
        if ($imgSrc) {
          $node.prepend($(`<img class="__image-lead__" src="${$imgSrc}"/>`));
        }
      },
    },

    // Is there anything that is in the result that shouldn't be?
    // The clean selectors will remove anything that matches from
    // the result
    clean: ['.pull-quote.pull-quote--large'],
  },
};


<|start_filename|>src/resource/utils/constants.js<|end_filename|>
import cheerio from 'cheerio';

// Browser does not like us setting user agent
export const REQUEST_HEADERS = cheerio.browser
  ? {}
  : {
      'User-Agent':
        'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36',
    };

// The number of milliseconds to attempt to fetch a resource before timing out.
export const FETCH_TIMEOUT = 10000;

// Content types that we do not extract content from
const BAD_CONTENT_TYPES = [
  'audio/mpeg',
  'image/gif',
  'image/jpeg',
  'image/jpg',
];

export const BAD_CONTENT_TYPES_RE = new RegExp(
  `^(${BAD_CONTENT_TYPES.join('|')})$`,
  'i'
);

// Use this setting as the maximum size an article can be
// for us to attempt parsing. Defaults to 5 MB.
export const MAX_CONTENT_LENGTH = 5242880;

// Turn the global proxy on or off
// Proxying is not currently enabled in Python source
// so not implementing logic in port.
export const PROXY_DOMAINS = false;
export const REQUESTS_PROXIES = {
  http: 'http://172.16.31.10:33333',
  https: 'http://172.16.31.10:33333',
};

export const DOMAINS_TO_PROXY = ['nih.gov', 'gutenberg.org'];


<|start_filename|>src/extractors/custom/www.bloomberg.com/index.js<|end_filename|>
export const WwwBloombergComExtractor = {
  domain: 'www.bloomberg.com',

  title: {
    selectors: [
      // normal articles
      '.lede-headline',

      // /graphics/ template
      'h1.article-title',

      // /news/ template
      'h1.lede-text-only__hed',
    ],
  },

  author: {
    selectors: [
      ['meta[name="parsely-author"]', 'value'],
      '.byline-details__link',

      // /graphics/ template
      '.bydek',

      // /news/ template
      '.author',
    ],
  },

  date_published: {
    selectors: [
      ['time.published-at', 'datetime'],
      ['time[datetime]', 'datetime'],
      ['meta[name="date"]', 'value'],
      ['meta[name="parsely-pub-date"]', 'value'],
    ],
  },

  dek: {
    selectors: [],
  },

  lead_image_url: {
    selectors: [['meta[name="og:image"]', 'value']],
  },

  content: {
    selectors: [
      '.article-body__content',

      // /graphics/ template
      ['section.copy-block'],

      // /news/ template
      '.body-copy',
    ],

    // Is there anything in the content you selected that needs transformed
    // before it's consumable content? E.g., unusual lazy loaded images
    transforms: {},

    // Is there anything that is in the result that shouldn't be?
    // The clean selectors will remove anything that matches from
    // the result
    clean: ['.inline-newsletter', '.page-ad'],
  },
};


<|start_filename|>src/cleaners/title.test.js<|end_filename|>
import assert from 'assert';
import cheerio from 'cheerio';

import HTML from './fixtures/html';
import { cleanTitle } from './index';

describe('cleanTitle(title, { url, $ })', () => {
  it('only uses h1 if there is only one on the page', () => {
    const title = 'Too Short';
    const $ = cheerio.load(HTML.docWith2H1s);

    assert.equal(cleanTitle(title, { url: '', $ }), title);
  });

  it('removes HTML tags from titles', () => {
    const $ = cheerio.load(HTML.docWithTagsInH1.before);
    const title = $('h1').html();

    assert.equal(cleanTitle(title, { url: '', $ }), HTML.docWithTagsInH1.after);
  });

  it('trims extraneous spaces', () => {
    const title = " This Is a Great Title That You'll Love ";
    const $ = cheerio.load(HTML.docWithTagsInH1.before);

    assert.equal(cleanTitle(title, { url: '', $ }), title.trim());
  });
});


<|start_filename|>src/extractors/generic/excerpt/extractor.test.js<|end_filename|>
import assert from 'assert';
import cheerio from 'cheerio';

import GenericExcerptExtractor, { clean } from './extractor';

describe('GenericExcerptExtractor', () => {
  describe('extract({ $, content, metaCache })', () => {
    it('returns og:description', () => {
      const actualExcerpt = 'Wow this is going to be something good.';
      const html = `
        <html>
          <head>
            <meta name="og:description" value="${actualExcerpt}" />
          </head>
        </html>
      `;
      const $ = cheerio.load(html);
      const metaCache = ['og:description'];

      const excerpt = GenericExcerptExtractor.extract({
        $,
        content: '',
        metaCache,
      });

      assert.equal(excerpt, actualExcerpt);
    });

    it('returns twitter:description', () => {
      const actualExcerpt = 'Wow this is going to be something good.';
      const html = `
        <html>
          <head>
            <meta name="twitter:description" value="${actualExcerpt}" />
          </head>
        </html>
      `;
      const $ = cheerio.load(html);
      const metaCache = ['twitter:description'];

      const excerpt = GenericExcerptExtractor.extract({
        $,
        content: '',
        metaCache,
      });

      assert.equal(excerpt, actualExcerpt);
    });

    it('falls back to the content', () => {
      const html = `
        <html>
          <head>
          </head>
        </html>
      `;
      const $ = cheerio.load(html);
      const content =
        '<div><p>Wow <b>this</b> is going to be something good.</p></div>';
      const metaCache = [];

      const excerpt = GenericExcerptExtractor.extract({
        $,
        content,
        metaCache,
      });

      assert.equal(excerpt, 'Wow this is going to be something good.');
    });
  });
});

describe('clean(text)', () => {
  it('truncates text longer than 200 chars and trims whitespance', () => {
    const longText = `
      Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor 
      incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis 
      nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. 
      Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu 
      fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in 
      culpa qui officia deserunt mollit anim id est laborum.
    `;
    const text = clean(longText);
    let shouldBe = `
      Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor
      incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud
      exercitation ullamco laboris nisi ut&hellip;
    `;
    shouldBe = shouldBe.replace(/[\s\n]+/g, ' ').trim();

    assert.equal(text, shouldBe);
  });
});


<|start_filename|>src/extractors/custom/mashable.com/index.js<|end_filename|>
export const MashableComExtractor = {
  domain: 'mashable.com',

  title: {
    selectors: ['h1.title'],
  },

  author: {
    selectors: ['span.author_name a'],
  },

  date_published: {
    selectors: [['meta[name="og:article:published_time"]', 'value']],
  },

  lead_image_url: {
    selectors: [['meta[name="og:image"]', 'value']],
  },

  content: {
    selectors: ['section.article-content.blueprint'],

    // Is there anything in the content you selected that needs transformed
    // before it's consumable content? E.g., unusual lazy loaded images
    transforms: {
      '.image-credit': 'figcaption',
    },

    // Is there anything that is in the result that shouldn't be?
    // The clean selectors will remove anything that matches from
    // the result
    clean: [],
  },
};


<|start_filename|>src/extractors/generic/title/constants.js<|end_filename|>
// TODO: It would be great if we could merge the meta and selector lists into
// a list of objects, because we could then rank them better. For example,
// .hentry .entry-title is far better suited than <meta title>.

// An ordered list of meta tag names that denote likely article titles. All
// attributes should be lowercase for faster case-insensitive matching. From
// most distinct to least distinct.
export const STRONG_TITLE_META_TAGS = [
  'tweetmeme-title',
  'dc.title',
  'rbtitle',
  'headline',
  'title',
];

// og:title is weak because it typically contains context that we don't like,
// for example the source site's name. Gotta get that brand into facebook!
export const WEAK_TITLE_META_TAGS = ['og:title'];

// An ordered list of XPath Selectors to find likely article titles. From
// most explicit to least explicit.
//
// Note - this does not use classes like CSS. This checks to see if the string
// exists in the className, which is not as accurate as .className (which
// splits on spaces/endlines), but for our purposes it's close enough. The
// speed tradeoff is worth the accuracy hit.
export const STRONG_TITLE_SELECTORS = [
  '.hentry .entry-title',
  'h1#articleHeader',
  'h1.articleHeader',
  'h1.article',
  '.instapaper_title',
  '#meebo-title',
];

export const WEAK_TITLE_SELECTORS = [
  'article h1',
  '#entry-title',
  '.entry-title',
  '#entryTitle',
  '#entrytitle',
  '.entryTitle',
  '.entrytitle',
  '#articleTitle',
  '.articleTitle',
  'post post-title',
  'h1.title',
  'h2.article',
  'h1',
  'html head title',
  'title',
];


<|start_filename|>src/utils/dom/convert-node-to.test.js<|end_filename|>
import assert from 'assert';
import cheerio from 'cheerio';

import convertNodeTo from './convert-node-to';

describe('convertNodeTo(node, $)', () => {
  it('takes a node and converts it to a diff tag', () => {
    const html = '<div>Should become a p</div>';
    const $ = cheerio.load(html);
    const node = $('div').first();

    const result = convertNodeTo(node, $).html();
    const after = '<p>Should become a p</p>';

    assert.equal(result, after);
  });

  it('retains attributes on conversion', () => {
    const html = '<span class="foo" score="100">Should keep its attrs</span>';
    const $ = cheerio.load(html);
    const node = $('span').first();

    const result = convertNodeTo(node, $, 'div').html();
    const after = '<div class="foo" score="100">Should keep its attrs</div>';

    assert.equal(result, after);
  });

  it('does nothing if node.get returns null', () => {
    const html = '<span class="foo" score="100">Should keep its attrs</span>';
    const $ = cheerio.load(html);
    const node = {
      get: () => null,
    };

    const result = convertNodeTo(node, $, 'div').html();

    assert.equal(result, html);
  });

  // In the browser, the contents of noscript tags aren't rendered, therefore
  // transforms on the noscript tag (commonly used for lazy-loading) don't work
  // as expected. This test case handles that
  it('handles noscript tags in the browser', () => {
    const html = '<noscript><img src="http://example.com" /></noscript>';
    const resultHtml = '<figure><img src="http://example.com"></figure>';
    const $ = cheerio.load(html);
    const node = $('noscript');

    const result = convertNodeTo(node, $, 'figure', 'noscript').html();

    assert.equal(result, resultHtml);
  });
});


<|start_filename|>src/extractors/generic/content/scoring/score-paragraph.test.js<|end_filename|>
import assert from 'assert';
import cheerio from 'cheerio';

import HTML from './fixtures/html';
import { scoreParagraph } from './index';

describe('Scoring utils', () => {
  describe('scoreParagraph(node)', () => {
    it('returns 0 if text is less than 25 chars', () => {
      const html = '<p><em>Foo</em> bar</p>';
      const $ = cheerio.load(html);
      const node = $('p').first();

      const score = scoreParagraph(node);

      assert.equal(score, 0);
    });

    it('returns 1 if text is > 25 chars and has 0 commas', () => {
      const $ = cheerio.load(HTML.score1);
      const node = $('p').first();

      const score = scoreParagraph(node);

      assert.equal(score, 1);
    });

    it('returns 3 if text is > 25 chars and has 2 commas', () => {
      const $ = cheerio.load(HTML.score3);
      const node = $('p').first();

      const score = scoreParagraph(node);

      assert.equal(score, 3);
    });

    it('returns 19 if text has 15 commas, ~600 chars', () => {
      const $ = cheerio.load(HTML.score19);
      const node = $('p').first();

      const score = scoreParagraph(node);

      assert.equal(score, 19);
    });
  });
});


<|start_filename|>src/utils/dom/clean-headers.test.js<|end_filename|>
import cheerio from 'cheerio';

import { assertClean } from 'test-helpers';

import HTML from './fixtures/html';
import { cleanHeaders } from './index';

describe('cleanHeaders(article, $)', () => {
  it('parses html and returns the article', () => {
    const $ = cheerio.load(HTML.cleanFirstHeds.before);

    const result = cleanHeaders($('*').first(), $);
    assertClean(result.html(), HTML.cleanFirstHeds.after);
  });

  it('removes headers when the header text matches the title', () => {
    const $ = cheerio.load(HTML.cleanTitleMatch.before);

    const result = cleanHeaders($('*').first(), $, 'Title Match');
    assertClean(result.html(), HTML.cleanTitleMatch.after);
  });

  it('removes headers with a negative weight', () => {
    const $ = cheerio.load(HTML.dropWithNegativeWeight.before);

    const result = cleanHeaders($('*').first(), $);
    assertClean(result.html(), HTML.dropWithNegativeWeight.after);
  });
});


<|start_filename|>src/utils/dom/convert-to-paragraphs.test.js<|end_filename|>
import cheerio from 'cheerio';

import { assertClean } from 'test-helpers';
import HTML from './fixtures/html';

import convertToParagraphs from './convert-to-paragraphs';

function assertBeforeAndAfter(key, fn) {
  const $ = cheerio.load(HTML[key].before);
  assertClean(fn($).html(), HTML[key].after);
}

describe('convertToParagraphs($)', () => {
  it('performs simple conversions', () => {
    // Skipping this one in the browser. It works, but since the browser wraps
    // elements in a div, the last span conversion won't work as expected.
    if (!cheerio.browser) {
      assertBeforeAndAfter('convertToParagraphs', convertToParagraphs);
    }
  });

  it('does not convert a div with nested p children', () => {
    const html = `
      <div>
        <div>
          <div>
            <p>This is a paragraph</p>
          </div>
        </div>
      </div>
    `;
    const $ = cheerio.load(html);
    assertClean(convertToParagraphs($).html(), html);
  });
});


<|start_filename|>src/extractors/generic/next-page-url/extractor.test.js<|end_filename|>
import assert from 'assert';
import cheerio from 'cheerio';

import GenericNextPageUrlExtractor from './extractor';

const fs = require('fs');

describe('GenericNextPageUrlExtractor', () => {
  it('returns most likely next page url', () => {
    const html = fs.readFileSync('./fixtures/ars.html', 'utf8');
    const $ = cheerio.load(html);
    const url =
      'http://arstechnica.com/gadgets/2016/08/the-connected-renter-how-to-make-your-apartment-smarter/';
    const next =
      'http://arstechnica.com/gadgets/2016/08/the-connected-renter-how-to-make-your-apartment-smarter/2';

    const nextPage = GenericNextPageUrlExtractor.extract({
      $,
      url,
    });

    assert.equal(nextPage, next);
  });

  it('returns null if there is no likely next page', () => {
    const html = '<div><p>HI</p></div>';
    const $ = cheerio.load(html);
    const url = 'http://example.com/foo/bar';

    const nextPage = GenericNextPageUrlExtractor.extract({
      $,
      url,
    });

    assert.equal(nextPage, null);
  });
});


<|start_filename|>src/extractors/custom/www.vox.com/index.js<|end_filename|>
export const WwwVoxComExtractor = {
  domain: 'www.vox.com',

  title: {
    selectors: ['h1.c-page-title'],
  },

  author: {
    selectors: [['meta[name="author"]', 'value']],
  },

  date_published: {
    selectors: [['meta[name="article:published_time"]', 'value']],
  },

  dek: {
    selectors: ['.p-dek'],
  },

  lead_image_url: {
    selectors: [['meta[name="og:image"]', 'value']],
  },

  content: {
    selectors: [
      ['figure.e-image--hero', '.c-entry-content'],
      '.c-entry-content',
    ],

    // Is there anything in the content you selected that needs transformed
    // before it's consumable content? E.g., unusual lazy loaded images
    transforms: {
      'figure .e-image__image noscript': $node => {
        const imgHtml = $node.html();
        $node
          .parents('.e-image__image')
          .find('.c-dynamic-image')
          .replaceWith(imgHtml);
      },

      'figure .e-image__meta': 'figcaption',
    },

    // Is there anything that is in the result that shouldn't be?
    // The clean selectors will remove anything that matches from
    // the result
    clean: [],
  },
};


<|start_filename|>src/utils/dom/set-attr.js<|end_filename|>
export default function setAttr(node, attr, val) {
  if (node.attribs) {
    node.attribs[attr] = val;
  } else if (node.attributes) {
    node.setAttribute(attr, val);
  }

  return node;
}


<|start_filename|>src/extractors/generic/content/scoring/find-top-candidate.test.js<|end_filename|>
import assert from 'assert';
import cheerio from 'cheerio';

import HTML from './fixtures/html';

import { getScore, findTopCandidate, scoreContent } from './index';

const fs = require('fs');

describe('findTopCandidate($)', () => {
  it('finds the top candidate from simple case', () => {
    const $ = cheerio.load(HTML.findDom1);

    const $$topCandidate = findTopCandidate($);

    assert.equal(getScore($$topCandidate), 100);
  });

  it('finds the top candidate from a nested case', () => {
    const $ = cheerio.load(HTML.findDom2);

    const $$topCandidate = findTopCandidate($);

    // this is wrapped in a div so checking
    // the score of the first child
    assert.equal(getScore($$topCandidate.first()), 50);
  });

  it('ignores tags like BR', () => {
    const $ = cheerio.load(HTML.findDom3);

    const $topCandidate = findTopCandidate($);

    assert.equal(getScore($topCandidate), 50);
  });

  it('returns BODY if no candidates found', () => {
    const $ = cheerio.load(HTML.topBody);

    const $topCandidate = findTopCandidate($);

    // browser won't allow body tag to be placed
    // arbitrarily/loaded on the page, so we tranform
    // it in cheerio-query, so this test would fail.
    if (!$.browser) {
      assert.equal($topCandidate.get(0).tagName, 'body');
    }
  });

  it('appends a sibling with a good enough score', () => {
    const html = fs.readFileSync('./fixtures/latimes.html', 'utf-8');

    let $ = cheerio.load(html);
    $ = scoreContent($);

    const $topCandidate = findTopCandidate($);
    assert.equal($($topCandidate).text().length, 3652);
  });
});


<|start_filename|>src/extractors/generic/content/extract-best-node.js<|end_filename|>
import { stripUnlikelyCandidates, convertToParagraphs } from 'utils/dom';

import { scoreContent, findTopCandidate } from './scoring';

// Using a variety of scoring techniques, extract the content most
// likely to be article text.
//
// If strip_unlikely_candidates is True, remove any elements that
// match certain criteria first. (Like, does this element have a
// classname of "comment")
//
// If weight_nodes is True, use classNames and IDs to determine the
// worthiness of nodes.
//
// Returns a cheerio object $
export default function extractBestNode($, opts) {
  if (opts.stripUnlikelyCandidates) {
    $ = stripUnlikelyCandidates($);
  }

  $ = convertToParagraphs($);
  $ = scoreContent($, opts.weightNodes);
  const $topCandidate = findTopCandidate($);

  return $topCandidate;
}
