Skip to content

Commit 1c66246

Browse files
fivefiltersfivefilters
committed
Consider #hash links in link density
mozilla/readability@3c83389 (not all these changes work as expected, so one section has been commented out for now)
1 parent 84a5622 commit 1c66246

File tree

6 files changed

+57
-10
lines changed

6 files changed

+57
-10
lines changed

‎src/Nodes/NodeTrait.php‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -238,19 +238,21 @@ public function getAllLinks()
238238
*/
239239
public function getLinkDensity()
240240
{
241-
$linkLength = 0;
242241
$textLength = mb_strlen($this->getTextContent(true));
243-
244-
if (!$textLength) {
242+
if ($textLength === 0) {
245243
return 0;
246244
}
247245

246+
$linkLength = 0;
247+
248248
$links = $this->getAllLinks();
249249

250250
if ($links) {
251251
/** @var DOMElement $link */
252252
foreach ($links as $link) {
253-
$linkLength += mb_strlen($link->getTextContent(true));
253+
$href = $link->getAttribute('href');
254+
$coefficient = ($href && preg_match(NodeUtility::$regexps['hashUrl'], $href)) ? 0.3 : 1;
255+
$linkLength += mb_strlen($link->getTextContent(true)) * $coefficient;
254256
}
255257
}
256258

‎src/Readability.php‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1882,8 +1882,6 @@ public function _cleanConditionally(DOMDocument $article, $tag)
18821882
return;
18831883
}
18841884

1885-
$isList = in_array($tag, ['ul', 'ol']);
1886-
18871885
/*
18881886
* Gather counts for other typical elements embedded within.
18891887
* Traverse backwards so we can remove nodes at the same time
@@ -1896,6 +1894,25 @@ public function _cleanConditionally(DOMDocument $article, $tag)
18961894
/** @var $node DOMElement */
18971895
$node = $allNodesWithTag[$length - 1 - $i];
18981896

1897+
$isList = in_array($tag, ['ul', 'ol']);
1898+
/*
1899+
// Doesn't seem to work as expected
1900+
// compared to JS version: https://github.com/mozilla/readability/commit/3c833899866ffb1f9130767110197fd6f5c08d4c
1901+
if (!$isList) {
1902+
$listLength = 0;
1903+
$listNodes = $this->_getAllNodesWithTag($node, ['ul', 'ol']);
1904+
array_walk($listNodes, function($list) use(&$listLength) {
1905+
$listLength += mb_strlen($list->getTextContent());
1906+
});
1907+
$nodeTextLength = mb_strlen($node->getTextContent());
1908+
if (!$nodeTextLength) {
1909+
$isList = true;
1910+
} else {
1911+
$isList = $listLength / $nodeTextLength > 0.9;
1912+
}
1913+
}
1914+
*/
1915+
18991916
// First check if this node IS data table, in which case don't remove it.
19001917
if ($tag === 'table' && $node->isReadabilityDataTable()) {
19011918
continue;

‎test/test-pages/aclu/expected-metadata.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"Author": "Daniel Kahn Gillmor",
3-
"Direction": "ltr",
3+
"Direction": null,
44
"Excerpt": "Facebook collects data about people who have never even opted in. But there are ways these non-users can protect themselves.",
55
"Image": "https:\/\/www.aclu.org\/sites\/default\/files\/styles\/metatag_og_image_1200x630\/public\/field_share_image\/web18-facebook-socialshare-1200x628-v02.png?itok=p77cQjOm",
66
"Title": "Facebook Is Tracking Me Even Though I\u2019m Not on Facebook",

‎test/test-pages/mozilla-1/expected.html‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,30 @@ <h2>Designed to <br>be redesigned</h2>
2323
<p><img src="http://mozorg.cdn.mozilla.net/media/img/firefox/desktop/customize/animations/flexible-bottom-fallback.cafd48a3d0a4.png" alt></p>
2424
</div>
2525
</div>
26-
26+
<div id="customize" data-ga-label="More ways to customize">
27+
<h2>More ways to customize</h2>
28+
29+
<ul id="customizer-list" role="tablist">
30+
<li> <a id="customize-themes" href="#themes">
31+
32+
Themes
33+
</a>
34+
35+
</li>
36+
<li> <a id="customize-addons" href="#add-ons">
37+
38+
Add-ons
39+
</a>
40+
41+
</li>
42+
<li> <a id="customize-awesomebar" href="#awesome-bar">
43+
44+
Awesome Bar
45+
</a>
46+
47+
</li>
48+
</ul>
49+
</div>
2750
<div id="customizers-wrapper">
2851
<div id="themes" role="tabpanel" aria-labelledby="customize-themes">
2952
<div>

‎test/test-pages/nytimes-2/expected.html‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,10 @@
3333
<p data-para-count="479" data-total-count="1333">It is hard to overestimate how complex an asset sale like this is. Some of the assets are self-contained, but they must be gathered up and transferred. Employees need to be shuffled around and compensation arrangements redone. Many contracts, like the now-infamous one struck with the search engine Mozilla, which <a href="http://www.recode.net/2016/7/7/12116296/marissa-mayer-deal-mozilla-yahoo-payment">may result in a payment of up to a $1 billion</a>, will contain change-of-control provisions that will be set off and have to be addressed. Tax issues always loom large.</p> <p><a href="#story-continues-1">Continue reading the main story</a>
3434
</p></div>
3535

36-
36+
<div id="story-continues-1">
37+
38+
<p><a href="#story-continues-2">Continue reading the main story</a></p>
39+
</div>
3740
<div>
3841
<p data-para-count="602" data-total-count="1935" id="story-continues-2">In the second step, at the closing, <a href="https://www.sec.gov/Archives/edgar/data/1011006/000119312516656036/d178500dex22.htm">Yahoo will sell the stock</a> in the single subsidiary to Verizon. At that point, Yahoo will change its name to something without “Yahoo” in it. My favorite is simply Remain Co., the name Yahoo executives are using. Remain Co. will become a holding company for the Alibaba and Yahoo Japan stock. Included will also be $10 billion in cash, plus the Excalibur patent portfolio and a number of minority investments including Snapchat. Ahh, if only Yahoo had bought Snapchat instead of Tumblr (indeed, if only Yahoo had bought Google or Facebook when it had the chance).</p>
3942

‎test/test-pages/wikipedia-2/expected.html‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -939,7 +939,9 @@ <h2>
939939
Historically, extractive industries have contributed strongly to New Zealand's economy, focussing at different times on sealing, whaling, <a href="http://fakehost/wiki/Phormium" title="Phormium">flax</a>, gold, <a href="http://fakehost/wiki/Kauri_gum" title="Kauri gum">kauri gum</a>, and native timber.<sup id="cite_ref-RWT_export_evolution_203-0"><a href="#cite_note-RWT_export_evolution-203">[196]</a></sup> The first shipment of refrigerated meat on the <i><a href="http://fakehost/wiki/Dunedin_(ship)" title="Dunedin (ship)">Dunedin</a></i> in 1882 led to the establishment of meat and dairy exports to Britain, a trade which provided the basis for strong economic growth in New Zealand.<sup id="cite_ref-204"><a href="#cite_note-204">[197]</a></sup> High demand for agricultural products from the United Kingdom and the United States helped New Zealanders achieve higher living standards than both Australia and Western Europe in the 1950s and 1960s.<sup id="cite_ref-205"><a href="#cite_note-205">[198]</a></sup> In 1973, New Zealand's export market was reduced when the United Kingdom joined the <a href="http://fakehost/wiki/European_Economic_Community" title="European Economic Community">European Economic Community</a><sup id="cite_ref-206"><a href="#cite_note-206">[199]</a></sup> and other compounding factors, such as the <a href="http://fakehost/wiki/1973_oil_crisis" title="1973 oil crisis">1973 oil</a> and <a href="http://fakehost/wiki/1979_oil_crisis" title="1979 oil crisis">1979 energy</a> crises, led to a severe <a href="http://fakehost/wiki/Depression_(economics)" title="Depression (economics)">economic depression</a>.<sup id="cite_ref-207"><a href="#cite_note-207">[200]</a></sup> Living standards in New Zealand fell behind those of Australia and Western Europe, and by 1982 New Zealand had the lowest per-capita income of all the developed nations surveyed by <a href="http://fakehost/wiki/World_Bank_Group" title="World Bank Group">the World Bank</a>.<sup id="cite_ref-208"><a href="#cite_note-208">[201]</a></sup> In the mid-1980s New Zealand deregulated its <a href="http://fakehost/wiki/Agriculture_in_New_Zealand" title="Agriculture in New Zealand">agricultural sector</a> by phasing out <a href="http://fakehost/wiki/Agricultural_subsidy" title="Agricultural subsidy">subsidies</a> over a three-year period.<sup id="cite_ref-209"><a href="#cite_note-209">[202]</a></sup><sup id="cite_ref-210"><a href="#cite_note-210">[203]</a></sup> Since 1984, successive governments engaged in major <a href="http://fakehost/wiki/Macroeconomic" title="Macroeconomic">macroeconomic</a> restructuring (known first as <a href="http://fakehost/wiki/Rogernomics" title="Rogernomics">Rogernomics</a> and then <a href="http://fakehost/wiki/Ruthanasia" title="Ruthanasia">Ruthanasia</a>), rapidly transforming New Zealand from a <a href="http://fakehost/wiki/Protectionism" title="Protectionism">protected</a> and highly regulated economy to a liberalised <a href="http://fakehost/wiki/Free-trade" title="Free-trade">free-trade</a> economy.<sup id="cite_ref-Liberalisation_211-0"><a href="#cite_note-Liberalisation-211">[204]</a></sup><sup id="cite_ref-212"><a href="#cite_note-212">[205]</a></sup>
940940
</p>
941941
<div>
942-
<p><a href="http://fakehost/wiki/File:MilfordSound.jpg"><img alt="Blue water against a backdrop of snow-capped mountains" src="http://upload.wikimedia.org/wikipedia/commons/thumb/4/42/MilfordSound.jpg/220px-MilfordSound.jpg" decoding="async" width="220" height="147" srcset="http://upload.wikimedia.org/wikipedia/commons/thumb/4/42/MilfordSound.jpg/330px-MilfordSound.jpg 1.5x, http://upload.wikimedia.org/wikipedia/commons/thumb/4/42/MilfordSound.jpg/440px-MilfordSound.jpg 2x" data-file-width="2048" data-file-height="1364"></a></p>
942+
<p><a href="http://fakehost/wiki/File:MilfordSound.jpg"><img alt="Blue water against a backdrop of snow-capped mountains" src="http://upload.wikimedia.org/wikipedia/commons/thumb/4/42/MilfordSound.jpg/220px-MilfordSound.jpg" decoding="async" width="220" height="147" srcset="http://upload.wikimedia.org/wikipedia/commons/thumb/4/42/MilfordSound.jpg/330px-MilfordSound.jpg 1.5x, http://upload.wikimedia.org/wikipedia/commons/thumb/4/42/MilfordSound.jpg/440px-MilfordSound.jpg 2x" data-file-width="2048" data-file-height="1364"></a></p><div>
943+
<p><a href="http://fakehost/wiki/Milford_Sound" title="Milford Sound">Milford Sound</a> is one of New Zealand's most famous tourist destinations.<sup id="cite_ref-213"><a href="#cite_note-213">[206]</a></sup>
944+
</p></div>
943945
</div>
944946
<p>
945947
Unemployment peaked above 10% in 1991 and 1992,<sup id="cite_ref-unemployment_214-0"><a href="#cite_note-unemployment-214">[207]</a></sup> following the <a href="http://fakehost/wiki/Black_Monday_(1987)" title="Black Monday (1987)">1987 share market crash</a>, but eventually fell to a record low (since 1986) of 3.7% in 2007 (ranking third from twenty-seven comparable OECD nations).<sup id="cite_ref-unemployment_214-1"><a href="#cite_note-unemployment-214">[207]</a></sup> However, the <a href="http://fakehost/wiki/Financial_crisis_of_2007%E2%80%932008" title="Financial crisis of 2007–2008">global financial crisis</a> that followed had a major impact on New Zealand, with the GDP shrinking for five consecutive quarters, the longest recession in over thirty years,<sup id="cite_ref-215"><a href="#cite_note-215">[208]</a></sup><sup id="cite_ref-216"><a href="#cite_note-216">[209]</a></sup> and unemployment rising back to 7% in late 2009.<sup id="cite_ref-217"><a href="#cite_note-217">[210]</a></sup> Unemployment rates for different age groups follow similar trends, but are consistently higher among youth. In the December 2014 quarter, the general unemployment rate was around 5.8%, while the unemployment rate for youth aged 15 to 21 was 15.6%.<sup id="cite_ref-unemployment_214-2"><a href="#cite_note-unemployment-214">[207]</a></sup> New Zealand has experienced a series of "<a href="http://fakehost/wiki/Brain_drain" title="Brain drain">brain drains</a>" since the 1970s<sup id="cite_ref-218"><a href="#cite_note-218">[211]</a></sup> that still continue today.<sup id="cite_ref-219"><a href="#cite_note-219">[212]</a></sup> Nearly one quarter of highly skilled workers live overseas, mostly in Australia and Britain, which is the largest proportion from any developed nation.<sup id="cite_ref-220"><a href="#cite_note-220">[213]</a></sup> In recent decades, however, a "brain gain" has brought in educated professionals from Europe and less developed countries.<sup id="cite_ref-221"><a href="#cite_note-221">[214]</a></sup><sup id="cite_ref-FOOTNOTEBain200644_222-0"><a href="#cite_note-FOOTNOTEBain200644-222">[215]</a></sup> Today New Zealand's economy benefits from a high level of <a href="http://fakehost/wiki/Innovation" title="Innovation">innovation</a>.<sup id="cite_ref-223"><a href="#cite_note-223">[216]</a></sup>

0 commit comments

Comments
 (0)