Skip to content

Commit 106943b

Browse files
Merge branch 'downloader'
2 parents dfba36d + 14f64de commit 106943b

File tree

4 files changed

+231
-4
lines changed

4 files changed

+231
-4
lines changed

cmd/golly/golly.go

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313
"strings"
1414
"syscall"
1515

16+
"github.com/matthewmueller/golly/internal/chrome"
1617
"github.com/matthewmueller/golly/internal/compiler/util"
1718

1819
kingpin "gopkg.in/alecthomas/kingpin.v2"
@@ -81,8 +82,18 @@ func run(ctx context.Context) error {
8182
}
8283
filePath := path.Join(cwd, *runFile)
8384

85+
root, err := util.GollyPath()
86+
if err != nil {
87+
return errors.Wrapf(err, "error getting root path")
88+
}
89+
90+
chromePath, err := chrome.Find(path.Join(root, "chrome"))
91+
if err != nil {
92+
return errors.Wrapf(err, "unable to get chrome path")
93+
}
94+
8495
result, err := api.Run(ctx, &api.RunSettings{
85-
ChromePath: os.Getenv("GOLLY_CHROME_PATH"),
96+
ChromePath: chromePath,
8697
FilePath: filePath,
8798
})
8899
if err != nil {

index.js

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
const puppeteer = require('puppeteer');
2+
3+
(async () => {
4+
const browser = await puppeteer.launch();
5+
const page = await browser.newPage();
6+
await page.goto('https://news.ycombinator.com', { waitUntil: 'networkidle2' });
7+
await page.pdf({ path: 'hn.pdf', format: 'A4' });
8+
9+
await browser.close();
10+
})();

internal/chrome/chrome.go

Lines changed: 34 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"net/url"
88
"os"
99
"os/exec"
10+
"path"
1011
"strconv"
1112
"strings"
1213
"syscall"
@@ -40,6 +41,31 @@ type Settings struct {
4041
Addr string
4142
}
4243

44+
var defaultFlags = []string{
45+
"--disable-background-networking",
46+
"--disable-background-timer-throttling",
47+
"--disable-client-side-phishing-detection",
48+
"--disable-default-apps",
49+
"--disable-extensions",
50+
"--disable-hang-monitor",
51+
"--disable-popup-blocking",
52+
"--disable-prompt-on-repost",
53+
"--disable-sync",
54+
"--disable-translate",
55+
"--metrics-recording-only",
56+
"--no-first-run",
57+
"--safebrowsing-disable-auto-update",
58+
59+
"--enable-automation",
60+
"--password-store=basic",
61+
"--use-mock-keychain",
62+
63+
"--headless",
64+
"--disable-gpu",
65+
"--hide-scrollbars",
66+
"--mute-audio",
67+
}
68+
4369
var errStopped = errors.New("stopped")
4470

4571
// New chrome
@@ -58,11 +84,16 @@ func New(parent context.Context, settings *Settings) (*Chrome, error) {
5884
// default flags
5985
flags := append(
6086
settings.Flags,
61-
"--headless",
62-
"--disable-gpu",
63-
"--remote-debugging-port="+addr.Port(),
87+
defaultFlags...,
6488
)
6589

90+
// user data dir
91+
tmp := os.TempDir()
92+
flags = append(flags, "--user-data-dir="+path.Join(tmp))
93+
94+
// remote debugging port
95+
flags = append(flags, "--remote-debugging-port="+addr.Port())
96+
6697
// create the command
6798
cmd := exec.CommandContext(ctx, settings.ExecutablePath, flags...)
6899

internal/chrome/downloader.go

Lines changed: 175 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,175 @@
1+
package chrome
2+
3+
import (
4+
"archive/zip"
5+
"bytes"
6+
"fmt"
7+
"io"
8+
"io/ioutil"
9+
"net/http"
10+
"os"
11+
"path"
12+
"path/filepath"
13+
"runtime"
14+
15+
"github.com/pkg/errors"
16+
)
17+
18+
// ChromiumRevision chromium revision
19+
const ChromiumRevision = "518818"
20+
21+
// DownloadHost download host
22+
const DownloadHost = "https://storage.googleapis.com"
23+
24+
var downloadURLs = map[string]string{
25+
"linux": "%s/chromium-browser-snapshots/Linux_x64/%s/chrome-linux.zip",
26+
"mac": "%s/chromium-browser-snapshots/Mac/%s/chrome-mac.zip",
27+
"win32": "%s/chromium-browser-snapshots/Win/%s/chrome-win32.zip",
28+
"win64": "%s/chromium-browser-snapshots/Win_x64/%s/chrome-win32.zip",
29+
}
30+
31+
// Find locally or remotely
32+
func Find(dir string) (string, error) {
33+
filepath, err := chromiumPath(dir)
34+
if err != nil {
35+
return "", errors.Wrapf(err, "error getting chromium path")
36+
}
37+
38+
if _, err := os.Stat(filepath); !os.IsNotExist(err) {
39+
return filepath, nil
40+
}
41+
42+
if err := download(dir); err != nil {
43+
return "", errors.Wrapf(err, "error downloading")
44+
}
45+
46+
return filepath, nil
47+
}
48+
49+
// Download path
50+
func download(dir string) error {
51+
var url, platform string
52+
53+
switch runtime.GOOS {
54+
case "darwin":
55+
platform = "mac"
56+
url = fmt.Sprintf(downloadURLs[platform], DownloadHost, ChromiumRevision)
57+
case "linux":
58+
platform = "linux"
59+
url = fmt.Sprintf(downloadURLs[platform], DownloadHost, ChromiumRevision)
60+
case "windows":
61+
platform = "windows"
62+
// TODO: 32 vs 64
63+
url = fmt.Sprintf(downloadURLs["win64"], DownloadHost, ChromiumRevision)
64+
default:
65+
return errors.New("unsupported operating system")
66+
}
67+
68+
res, err := http.Get(url)
69+
if err != nil {
70+
return errors.Wrapf(err, "error getting download")
71+
}
72+
defer res.Body.Close()
73+
74+
src, err := ioutil.ReadAll(res.Body)
75+
if err != nil {
76+
return err
77+
}
78+
79+
folderPath := path.Join(dir, platform+"-"+ChromiumRevision)
80+
if err := os.MkdirAll(folderPath, 0775); err != nil {
81+
return errors.Wrapf(err, "error making directory")
82+
}
83+
84+
return unzip(src, folderPath)
85+
}
86+
87+
// Path to the executable
88+
// TODO: test on windows
89+
func chromiumPath(dir string) (string, error) {
90+
var platform string
91+
switch runtime.GOOS {
92+
case "darwin":
93+
platform = "mac"
94+
return path.Join(dir, platform+"-"+ChromiumRevision, "chrome-mac", "Chromium.app", "Contents", "MacOS", "Chromium"), nil
95+
case "linux":
96+
platform = "linux"
97+
return path.Join(dir, platform+"-"+ChromiumRevision, "chrome-linux", "chrome"), nil
98+
case "windows":
99+
platform = "windows"
100+
return path.Join(dir, platform+"-"+ChromiumRevision, "chrome-win32", "chrome.exe"), nil
101+
default:
102+
return "", errors.New("unsupported operating system")
103+
}
104+
}
105+
106+
// Unzip a file to a destination and preserve the symlinks
107+
// Based on http://stackoverflow.com/a/24792688/842097 with symlink additions
108+
func unzip(src []byte, dest string) error {
109+
rdr := bytes.NewReader(src)
110+
111+
r, err := zip.NewReader(rdr, rdr.Size())
112+
if err != nil {
113+
return err
114+
}
115+
116+
os.MkdirAll(dest, 0755)
117+
118+
// Closure to address file descriptors issue with all the deferred .Close() methods
119+
extractAndWriteFile := func(f *zip.File) error {
120+
rc, err := f.Open()
121+
if err != nil {
122+
return err
123+
}
124+
defer func() {
125+
if err := rc.Close(); err != nil {
126+
panic(err)
127+
}
128+
}()
129+
130+
path := filepath.Join(dest, f.Name)
131+
132+
if f.FileInfo().IsDir() {
133+
os.MkdirAll(path, f.Mode())
134+
} else if f.FileInfo().Mode()&os.ModeSymlink != 0 {
135+
buffer := make([]byte, f.FileInfo().Size())
136+
size, err := rc.Read(buffer)
137+
if err != nil {
138+
return err
139+
}
140+
141+
target := string(buffer[:size])
142+
143+
err = os.Symlink(target, path)
144+
if err != nil {
145+
return err
146+
}
147+
} else {
148+
os.MkdirAll(filepath.Dir(path), f.Mode())
149+
f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
150+
if err != nil {
151+
return err
152+
}
153+
defer func() {
154+
if err = f.Close(); err != nil {
155+
panic(err)
156+
}
157+
}()
158+
159+
_, err = io.Copy(f, rc)
160+
if err != nil {
161+
return err
162+
}
163+
}
164+
return nil
165+
}
166+
167+
for _, f := range r.File {
168+
err := extractAndWriteFile(f)
169+
if err != nil {
170+
return err
171+
}
172+
}
173+
174+
return nil
175+
}

0 commit comments

Comments
 (0)