Skip to content

Commit c6c5f7d

Browse files
harlan-zwharlan-zw
committed
feat: userAgent config
Fixes #243
1 parent 1bb0095 commit c6c5f7d

File tree

8 files changed

+60
-33
lines changed

8 files changed

+60
-33
lines changed

‎docs/content/2.integrations/0.cli.md

+26-25
Original file line numberDiff line numberDiff line change
@@ -36,32 +36,33 @@ See the [Configuration](#configuration) section for more details and the guides.
3636

3737
### CLI Options
3838

39-
| Options | |
40-
|------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
41-
| `-v, --version` | Display version number. |
42-
| `--site <url>` | Host URL to scan. |
43-
| `--root <path>` | Define the project root. |
44-
| `--config-file <path>` | Path to config file. |
45-
| `--output-path <path>` | Path to save the contents of the client and reports to. |
46-
| `--cache` | Enable the caching. |
47-
| `--no-cache` | Disable the caching. |
48-
| `--desktop` | Simulate device as desktop. |
49-
| `--mobile` | Simulate device as mobile. |
50-
| `--throttle` | Enable the throttling. |
51-
| `--samples` | Specify the amount of samples to run. |
52-
| `--sitemaps` | Comma separated list of sitemaps to use for scanning. |
53-
| `--urls` | Specify explicit relative paths as a comma-separated list.<br>e.g. `unlighthouse --site unlighthouse.dev --urls /guide,/api,/config` |
39+
| Options | |
40+
|------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
41+
| `-v, --version` | Display version number. |
42+
| `--site <url>` | Host URL to scan. |
43+
| `--root <path>` | Define the project root. |
44+
| `--config-file <path>` | Path to config file. |
45+
| `--output-path <path>` | Path to save the contents of the client and reports to. |
46+
| `--cache` | Enable the caching. |
47+
| `--no-cache` | Disable the caching. |
48+
| `--desktop` | Simulate device as desktop. |
49+
| `--mobile` | Simulate device as mobile. |
50+
| `--throttle` | Enable the throttling. |
51+
| `--samples` | Specify the amount of samples to run. |
52+
| `--sitemaps` | Comma separated list of sitemaps to use for scanning. |
53+
| `--urls` | Specify explicit relative paths as a comma-separated list.<br>e.g. `unlighthouse --site unlighthouse.dev --urls /guide,/api,/config` |
5454
| `--exclude-urls` | Specify relative paths (string or regex) to exclude from scanning as a comma-separated list. <br>e.g. `unlighthouse --site unlighthouse.dev --exclude-urls /guide/.*,/api/.*` |
55-
| `--include-urls` | Specify relative paths (string or regex) to include as a comma-separated list. <br>e.g. `unlighthouse --site unlighthouse.dev --include-urls /guide/.*` |
56-
| `--enable-javascript` | When inspecting the HTML wait for the javascript to execute. Useful for SPAs. |
57-
| `--disable-javascript` | When inspecting the HTML, don't wait for the javascript to execute. |
58-
| `--enable-i18n-pages` | Enable scanning pages which use x-default. |
59-
| `--disable-i18n-pages` | Disable scanning pages which use x-default. |
60-
| `--disable-dynamic-sampling` | Disable dynamic sampling of paths. |
61-
| `--extra-headers` | Extra headers to send with the requests. |
62-
| `--default-query-params` | Default query params to send with the requests. |
63-
| `-d, --debug` | Debug. Enable debugging in the logger. |
64-
| `-h, --help` | Display available CLI options |
55+
| `--include-urls` | Specify relative paths (string or regex) to include as a comma-separated list. <br>e.g. `unlighthouse --site unlighthouse.dev --include-urls /guide/.*` |
56+
| `--enable-javascript` | When inspecting the HTML wait for the javascript to execute. Useful for SPAs. |
57+
| `--disable-javascript` | When inspecting the HTML, don't wait for the javascript to execute. |
58+
| `--enable-i18n-pages` | Enable scanning pages which use x-default. |
59+
| `--disable-i18n-pages` | Disable scanning pages which use x-default. |
60+
| `--disable-dynamic-sampling` | Disable dynamic sampling of paths. |
61+
| `--extra-headers` | Extra headers to send with the requests. |
62+
| `--user-agent` | Provide a custom user agent for all network requests. |
63+
| `--default-query-params` | Default query params to send with the requests. |
64+
| `-d, --debug` | Debug. Enable debugging in the logger. |
65+
| `-h, --help` | Display available CLI options |
6566

6667
### Config File
6768

‎docs/content/3.api/config.md

+7
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,13 @@ Provide cookies to be set for Axios and Puppeteer requests.
8585

8686
Provide extra headers to be set for Axios and Puppeteer requests.
8787

88+
### userAgent
89+
90+
- **Type:** `string`
91+
- **Default:** `undefined`
92+
93+
Provide a custom user agent for all network requests.
94+
8895
### defaultQueryParams
8996

9097
- **Type:** ` false | QueryObject`

‎packages/cli/src/createCli.ts

+1
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ export default function createCli() {
2020
cli.option('--mobile', 'Simulate device as mobile.')
2121

2222
cli.option('--site <site>', 'Host URL to scan.')
23+
cli.option('--user-agent <user-agent>', 'Specify a top-level user agent all requests will use.')
2324
cli.option('--router-prefix <site>', 'The URL path prefix for the client and API to run from.')
2425
cli.option('--sitemaps <sitemaps>', 'Comma separated list of sitemaps to use for scanning. Providing these will override any in robots.txt.')
2526
cli.option('--samples <samples>', 'Specify the amount of samples to run.')

‎packages/cli/src/types.ts

+1
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ export interface CliOptions {
3030
disableSitemap?: boolean
3131
disableDynamicSampling?: boolean
3232
sitemaps?: string
33+
userAgent?: string
3334
}
3435

3536
export interface CiOptions extends CliOptions {

‎packages/cli/src/util.ts

+10
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,16 @@ export function pickOptions(options: CiOptions | CliOptions): UserConfig {
143143
})
144144
}
145145

146+
if (options.userAgent) {
147+
picked.extraHeaders = picked.extraHeaders || {}
148+
picked.extraHeaders['User-Agent'] = options.userAgent
149+
// set lighthouse
150+
picked.lighthouseOptions = picked.lighthouseOptions || {}
151+
picked.lighthouseOptions.emulatedUserAgent = options.userAgent
152+
// pupeteer will respect userAgent
153+
picked.userAgent = options.userAgent
154+
}
155+
146156
if (options.defaultQueryParams) {
147157
picked.defaultQueryParams = picked.defaultQueryParams || {}
148158
options.defaultQueryParams.split(',').forEach((param) => {

‎packages/core/src/puppeteer/util.ts

+3
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,9 @@ export async function setupPage(page: Page) {
5959
await page.setExtraHTTPHeaders(resolvedConfig.extraHeaders)
6060
.catch(softErrorHandler('Failed to set extra headers'))
6161
}
62+
if (resolvedConfig.userAgent) {
63+
await page.setUserAgent(resolvedConfig.userAgent)
64+
}
6265
await hooks.callHook('puppeteer:before-goto', page)
6366
}
6467
})

‎packages/core/src/types.ts

+6
Original file line numberDiff line numberDiff line change
@@ -291,6 +291,12 @@ export interface ResolvedUserConfig {
291291
* @default false
292292
*/
293293
extraHeaders: false | Record<string, string>
294+
/**
295+
* The user agent to use for all requests. Uses default puppeteer / lighthouse user agent if not provided.
296+
*
297+
* @default false
298+
*/
299+
userAgent?: string
294300
/**
295301
* Query params to add to every request.
296302
*

‎packages/core/src/util.ts

+6-8
Original file line numberDiff line numberDiff line change
@@ -146,21 +146,19 @@ export async function createAxiosInstance(resolvedConfig: ResolvedUserConfig) {
146146

147147
axiosOptions.headers = axiosOptions.headers || {}
148148
// this should always be set
149-
axiosOptions.headers['User-Agent'] = resolvedConfig.lighthouseOptions.emulatedUserAgent || 'Unlighthouse'
150149

151150
if (resolvedConfig.cookies) {
152151
axiosOptions.headers.Cookie = resolvedConfig.cookies
153152
.map(cookie => `${cookie.name}=${cookie.value}`)
154153
.join('; ')
155154
}
156155

157-
if (resolvedConfig.extraHeaders) {
158-
axiosOptions.headers = {
159-
// fallback user agent, allow overriding
160-
'User-Agent': 'Unlighthouse',
161-
...resolvedConfig.extraHeaders,
162-
...axiosOptions.headers,
163-
}
156+
const userAgent = resolvedConfig.userAgent || resolvedConfig.lighthouseOptions.emulatedUserAgent || 'Unlighthouse'
157+
axiosOptions.headers = {
158+
// fallback user agent, allow overriding
159+
'User-Agent': userAgent,
160+
...(resolvedConfig.extraHeaders || {}),
161+
...axiosOptions.headers,
164162
}
165163

166164
if (resolvedConfig.defaultQueryParams)

0 commit comments

Comments
 (0)