Skip to content

Commit b10eabb

Browse files
feat: add metascraper-imdb (#876)
* feat: add metascraper-imdb IMDb Open Graph is title+stars+genre and runtime; JSON-LD has the real title, plot, and director. Co-authored-by: Cursor <[email protected]> * test: repair postplanner favicon drift HubSpot's favicon 404s, so logo-favicon falls back to Google. The gstatic host (t2 vs t3) is load-balanced, so assert that class of URL instead of snapshotting the exact host. Co-authored-by: Cursor <[email protected]> --------- Co-authored-by: Cursor <[email protected]>
1 parent 63168a1 commit b10eabb

13 files changed

Lines changed: 237 additions & 3 deletions

File tree

‎README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -264,6 +264,7 @@ const metascraper = require('metascraper')([
264264
- [metascraper-amazon](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-amazon) – Metascraper integration for Amazon.
265265
- [metascraper-bluesky](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-bluesky) – Metascraper integration for Bluesky.
266266
- [metascraper-dribbble](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-dribbble) – Metascraper integration for Dribble.
267+
- [metascraper-imdb](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-imdb) – Metascraper integration for IMDb.
267268
- [metascraper-instagram](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-instagram) – Metascraper integration for Instagram.
268269
- [metascraper-reddit](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-reddit) – Metascraper integration for Reddit.
269270
- [metascraper-soundcloud](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-soundcloud) – Metascraper integration for SoundCloud.
Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
<div align="center">
2+
<br>
3+
<img style="width: 500px; margin:3rem 0 1.5rem;" src="https://metascraper.js.org/static/logo-banner.png" alt="metascraper">
4+
<br>
5+
<br>
6+
<p align="center"><strong>metascraper-imdb</strong>: Metascraper integration with IMDb.</p>
7+
<p align="center">See our <a href="https://metascraper.js.org" target='_blank' rel='noopener noreferrer'>website</a> for more information.</p>
8+
<br>
9+
</div>
10+
11+
## Install
12+
13+
```bash
14+
$ npm install metascraper-imdb --save
15+
```
16+
17+
## License
18+
19+
**metascraper-imdb** © [Microlink](https://microlink.io), released under the [MIT](https://github.com/microlinkhq/metascraper/blob/master/LICENSE.md) License.<br>
20+
Authored and maintained by [Microlink](https://microlink.io) with help from [contributors](https://github.com/microlinkhq/metascraper/contributors).
21+
22+
> [microlink.io](https://microlink.io) · GitHub [microlinkhq](https://github.com/microlinkhq) · X [@microlinkhq](https://x.com/microlinkhq)
Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
{
2+
"name": "metascraper-imdb",
3+
"description": "Metascraper rules tailored for IMDb pages — richer metadata than generic HTML parsers.",
4+
"homepage": "https://github.com/microlinkhq/metascraper/packages/metascraper-imdb",
5+
"version": "5.56.2",
6+
"types": "src/index.d.ts",
7+
"main": "src/index.js",
8+
"author": {
9+
"email": "[email protected]",
10+
"name": "microlink.io",
11+
"url": "https://microlink.io"
12+
},
13+
"repository": {
14+
"directory": "packages/metascraper-imdb",
15+
"type": "git",
16+
"url": "git+https://github.com/microlinkhq/metascraper.git"
17+
},
18+
"bugs": {
19+
"url": "https://github.com/microlinkhq/metascraper/issues"
20+
},
21+
"keywords": [
22+
"extract",
23+
"film",
24+
"html",
25+
"imdb",
26+
"metadata",
27+
"metascraper",
28+
"movie",
29+
"preview",
30+
"scraper",
31+
"tv"
32+
],
33+
"dependencies": {
34+
"@metascraper/helpers": "workspace:*"
35+
},
36+
"devDependencies": {
37+
"ava": "8"
38+
},
39+
"engines": {
40+
"node": ">= 22"
41+
},
42+
"files": [
43+
"src"
44+
],
45+
"scripts": {
46+
"test": "NODE_PATH=.. TZ=UTC ava --timeout 15s"
47+
},
48+
"license": "MIT"
49+
}
Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
declare function rules(): import('metascraper').Rules;
2+
export = rules;
Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
'use strict'
2+
3+
const {
4+
$jsonld,
5+
$meta,
6+
author,
7+
description,
8+
memoizeOne,
9+
parseUrl,
10+
title,
11+
toRule
12+
} = require('@metascraper/helpers')
13+
14+
const toAuthor = toRule(author)
15+
const toDescription = toRule(description)
16+
const toTitle = toRule(title)
17+
18+
const test = memoizeOne(url => parseUrl(url).domain === 'imdb.com')
19+
20+
module.exports = () => {
21+
const rules = {
22+
author: [
23+
toAuthor($jsonld('director.name')),
24+
toAuthor($jsonld('creator.name'))
25+
],
26+
description: [
27+
toDescription($meta('description')),
28+
toDescription($jsonld('description'))
29+
],
30+
publisher: () => 'IMDb',
31+
title: [toTitle($jsonld('name'))]
32+
}
33+
34+
rules.test = ({ url }) => test(url)
35+
36+
rules.pkgName = 'metascraper-imdb'
37+
38+
return rules
39+
}
40+
41+
module.exports.test = test
Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
<!DOCTYPE html>
2+
<html lang="en">
3+
<head>
4+
<title>Ferris Bueller's Day Off (1986) - IMDb</title>
5+
<meta name="description" content="Ferris Bueller's Day Off: Directed by John Hughes. With Matthew Broderick, Alan Ruck, Mia Sara, Jeffrey Jones. A brash, cocky high school senior, tired of skipping school to spend a boring day at home, is determined to enjoy an epic day roaring around his favorite Chicago sites, enlisting his best friend and girlfriend to join him on the adventure.">
6+
<meta property="og:site_name" content="IMDb">
7+
<meta property="og:title" content="Ferris Bueller's Day Off (1986) ⭐ 7.8 | Comedy">
8+
<meta property="og:description" content="1h 43m | PG-13">
9+
<meta property="og:image" content="https://m.media-amazon.com/images/M/MV5BZWYwMjUxNjMtMzE0MC00NDM3LWIxMmQtYmEyNWVjNjdlZGZjXkEyXkFqcGc@._V1_FMjpg_UX1000_.jpg">
10+
<script type="application/ld+json">
11+
{
12+
"@context": "https://schema.org",
13+
"@type": "Movie",
14+
"url": "https://www.imdb.com/title/tt0091042/",
15+
"name": "Ferris Bueller&apos;s Day Off",
16+
"image": "https://m.media-amazon.com/images/M/MV5BZWYwMjUxNjMtMzE0MC00NDM3LWIxMmQtYmEyNWVjNjdlZGZjXkEyXkFqcGc@._V1_.jpg",
17+
"description": "A brash, cocky high school senior, tired of skipping school to spend a boring day at home, is determined to enjoy an epic day roaring around his favorite Chicago sites, enlisting his best friend and girlfriend to join him on the adventure.",
18+
"review": {
19+
"@type": "Review",
20+
"itemReviewed": { "@type": "Movie", "url": "https://www.imdb.com/title/tt0091042/" },
21+
"author": { "@type": "Person", "name": "mdm-11" },
22+
"name": "Smart A*s v. The Establishment -- Ferris Bueller Rules!"
23+
},
24+
"aggregateRating": { "@type": "AggregateRating", "ratingValue": 7.8 },
25+
"contentRating": "PG-13",
26+
"genre": ["Comedy"],
27+
"datePublished": "1986-06-11",
28+
"director": [{ "@type": "Person", "url": "https://www.imdb.com/name/nm0000455/", "name": "John Hughes" }],
29+
"creator": [
30+
{ "@type": "Organization", "url": "https://www.imdb.com/company/co0023400/" },
31+
{ "@type": "Person", "url": "https://www.imdb.com/name/nm0000455/", "name": "John Hughes" }
32+
]
33+
}
34+
</script>
35+
</head>
36+
<body></body>
37+
</html>
Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
'use strict'
2+
3+
const { readFile } = require('fs/promises')
4+
const { resolve } = require('path')
5+
const test = require('ava').default
6+
7+
const metascraperImdb = require('metascraper-imdb')
8+
9+
const createMetascraper = (...args) =>
10+
require('metascraper')([
11+
metascraperImdb(...args),
12+
require('metascraper-author')(),
13+
require('metascraper-date')(),
14+
require('metascraper-image')(),
15+
require('metascraper-description')(),
16+
require('metascraper-lang')(),
17+
require('metascraper-publisher')(),
18+
require('metascraper-title')(),
19+
require('metascraper-url')()
20+
])
21+
22+
test('from a title page', async t => {
23+
const url = 'https://www.imdb.com/title/tt0091042/'
24+
const html = await readFile(resolve(__dirname, 'fixtures/title.html'))
25+
const metascraper = createMetascraper()
26+
const metadata = await metascraper({ url, html })
27+
t.is(metadata.title, 'Ferris Bueller’s Day Off')
28+
t.is(metadata.author, 'John Hughes')
29+
t.true(
30+
metadata.description.startsWith(
31+
'Ferris Bueller’s Day Off: Directed by John Hughes'
32+
)
33+
)
34+
t.false(metadata.title.includes('⭐'))
35+
t.false(metadata.description.includes('1h 43m'))
36+
t.not(metadata.author, 'mdm-11')
37+
t.is(metadata.publisher, 'IMDb')
38+
t.snapshot(metadata)
39+
})
Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
# Snapshot report for `test/index.js`
2+
3+
The actual snapshot is saved in `index.js.snap`.
4+
5+
Generated by [AVA](https://avajs.dev).
6+
7+
## from a title page
8+
9+
> Snapshot 1
10+
11+
{
12+
author: 'John Hughes',
13+
date: '1986-06-11T00:00:00.000Z',
14+
description: 'Ferris Bueller’s Day Off: Directed by John Hughes. With Matthew Broderick, Alan Ruck, Mia Sara, Jeffrey Jones. A brash, cocky high school senior, tired of skipping school to spend a boring day at home, is determined to enjoy an epic day roaring around his favorite Chicago sites, enlisting his best friend and girlfriend to join him on the adventure.',
15+
image: 'https://m.media-amazon.com/images/M/MV5BZWYwMjUxNjMtMzE0MC00NDM3LWIxMmQtYmEyNWVjNjdlZGZjXkEyXkFqcGc@._V1_FMjpg_UX1000_.jpg',
16+
lang: 'en',
17+
publisher: 'IMDb',
18+
title: 'Ferris Bueller’s Day Off',
19+
url: 'https://www.imdb.com/title/tt0091042/',
20+
}
676 Bytes
Binary file not shown.
Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
'use strict'
2+
3+
const test = require('ava').default
4+
5+
const { test: validator } = require('..')
6+
7+
test('true', t => {
8+
t.true(validator('https://www.imdb.com/title/tt0091042/'))
9+
t.true(validator('https://imdb.com/title/tt0091042/'))
10+
t.true(validator('https://m.imdb.com/title/tt0091042/'))
11+
t.true(validator('https://www.imdb.com/name/nm0000111/'))
12+
})
13+
14+
test('false', t => {
15+
t.false(
16+
validator(
17+
'https://soundcloud.com/beautybrainsp/beauty-brain-swag-bandicoot'
18+
)
19+
)
20+
t.false(validator('https://www.rottentomatoes.com/m/ferris_buellers_day_off'))
21+
t.false(validator('https://imdb.evil.com/title/tt0091042/'))
22+
t.false(validator('https://notimdb.com/title/tt0091042/'))
23+
})

0 commit comments

Comments
 (0)