diff --git a/.vscode/launch.json b/.vscode/launch.json
index 64da791f43dae35c75152ed020298e86db9a77c9..d5d185cec54649405ca688b1466d8f2e4447fd09 100644
--- a/.vscode/launch.json
+++ b/.vscode/launch.json
@@ -25,7 +25,14 @@
"program": "${workspaceFolder}/dist/index.js",
"args": [
"inspect",
- "https://dbogatov.org"
+ "https://dbogatov.org",
+ "-r",
+ "-t",
+ "100",
+ "--ignore-prefixes",
+ "m,l",
+ "--accept-codes",
+ "444,555"
],
"console": "integratedTerminal",
"internalConsoleOptions": "neverOpen",
diff --git a/src/index.ts b/src/index.ts
index 5fd1034d1ad50997256dd3ad900390e121dbd047..dcca8989fd6a8e5bb31be661b8263d94bb1416af 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -11,9 +11,18 @@ commander
.command("inspect <url>")
.description("Check links in the given URL")
.option("-r, --recursive", "recursively check all links in all URLs within supplied host", false)
+ .option("-t, --timeout <number>", "timeout in ms after which the link will be considered broken", (value: string, _) => parseInt(value), 2000)
+ .option("--ignore-prefixes <coma-separated-strings>", "prefix(es) to ignore (without ':'), like mailto: and tel:", (value: string, _) => value.split(","), ["mailto", "tel"])
+ .option("--accept-codes <coma-separated-numbers>", "HTTP response code(s) (beyond 200-299) to accept, like 999 for linkedin", (value: string, _) => value.split(",").map(code => parseInt(code)), [999])
+ .option("-s, --skip <globs>", "URLs to skip defined by globs, like '*linkedin*'", (value: string, previous: string[]) => previous.concat([value]), [])
.action(async (url: string, inspectObj) => {
- let inspector = new Inspector(new URLsMatchingSet(), new Config())
+ let inspector = new Inspector(new URLsMatchingSet(), {
+ acceptedCodes: inspectObj.acceptCodes as number[],
+ timeout: parseInt(inspectObj.timeout as string),
+ ignoredPrefixes: inspectObj.ignorePrefixes as string[],
+ skipURLs: inspectObj.skip as string[]
+ })
let result = await inspector.processURL(new URL(url), inspectObj.recursive)
let success = result.report(new ConsoleReporter())
diff --git a/src/inspector.ts b/src/inspector.ts
index b97e1cd86bf89cfb59bdd2e46371f94f533961fb..1e6e1afb1eea6fe8922c7ff1ad9fc49125680cad 100644
--- a/src/inspector.ts
+++ b/src/inspector.ts
@@ -1,6 +1,7 @@
import * as parser from "htmlparser2"
import axios, { AxiosError } from "axios"
import { Result, CheckStatus } from "./result";
+import { performance } from "perf_hooks"
export class Inspector {
@@ -19,14 +20,27 @@ export class Inspector {
try {
url = parent ? new URL(url, parent).href : url
- if (result.isChecked(url) || this.config.ignoredExtensions.some(ext => url.startsWith(ext + ":"))) {
- result.add({ url: url, status: CheckStatus.Skipped }, parent)
+ if (result.isChecked(url) || this.config.ignoredPrefixes.some(ext => url.startsWith(ext + ":"))) {
+ result.add({ url: url, status: CheckStatus.Skipped, duration: 0 }, parent)
} else {
- const response = await axios.get(parent ? new URL(url, parent).href : url, { timeout: this.config.timeout })
+ const instance = axios.create()
+ instance.interceptors.request.use(config => {
+ config.headers["request-startTime"] = performance.now()
+ return config
+ })
+ instance.interceptors.response.use((response) => {
+ const start = response.config.headers["request-startTime"]
+ const end = performance.now()
+ response.headers["request-duration"] = end - start
+ return response
+ })
+ const response = await instance.get(parent ? new URL(url, parent).href : url, { timeout: this.config.timeout })
+ const duration = response.headers["request-duration"]
+
let html = response.data as string
- if (url == originalUrl.href || (recursive && originalUrl.host == new URL(url).host)) {
+ if (url == originalUrl.href || (recursive && originalUrl.origin == new URL(url).origin)) {
let discoveredURLs = this.extractURLs(html)
@@ -35,20 +49,23 @@ export class Inspector {
}
}
- result.add({ url: url, status: CheckStatus.OK }, parent)
+ result.add({ url: url, status: CheckStatus.OK, duration: duration }, parent)
}
} catch (exception) {
-
const error: AxiosError = exception;
- if (!error.response) {
- result.add({ url: url, status: CheckStatus.GenericError }, parent)
+ if ((exception.message as string).includes("timeout")) {
+ result.add({ url: url, status: CheckStatus.Timeout, duration: this.config.timeout }, parent)
+ } else if (!error.response) {
+ result.add({ url: url, status: CheckStatus.GenericError, duration: 0 }, parent)
} else {
+ const duration = performance.now() - error.response.config.headers["request-startTime"]
+
if (this.config.acceptedCodes.some(code => code == error.response?.status)) {
- result.add({ url: url, status: CheckStatus.OK }, parent)
+ result.add({ url: url, status: CheckStatus.OK, duration: duration }, parent)
} else {
- result.add({ url: url, status: CheckStatus.NonSuccessCode, message: `${error.response.status}` }, parent)
+ result.add({ url: url, status: CheckStatus.NonSuccessCode, message: `${error.response.status}`, duration: duration }, parent)
}
}
}
@@ -96,9 +113,10 @@ export class Inspector {
export class Config {
- public acceptedCodes: number[] = [999]
- public timeout: number = 2000
- public ignoredExtensions: string[] = ["mailto", "tel"]
+ acceptedCodes: number[] = [999]
+ timeout: number = 2000
+ ignoredPrefixes: string[] = ["mailto", "tel"]
+ skipURLs: string[] = []
}
export enum URLMatchingRule {
diff --git a/src/report.ts b/src/report.ts
index c92af94fa6b777954f44882c89d36f8bfb3f1aae..5f8da3eb535dd22b00bbe79a8636f6d3086f39fb 100644
--- a/src/report.ts
+++ b/src/report.ts
@@ -13,11 +13,11 @@ export class ConsoleReporter implements IReporter {
console.log(`${indent ? "\t" : ""}${chalk.green(`OK: ${oks}`)}, ${chalk.grey(`skipped: ${skipped}`)}, ${chalk.red(`broken: ${broken}`)}`)
}
- private printCheck(status: CheckStatus, url: string) {
+ private printCheck(check: ResultItem) {
let statusLabel: string
const labelWidth = 7
- switch (status) {
+ switch (check.status) {
case CheckStatus.OK:
statusLabel = chalk.green("OK".padEnd(labelWidth))
break;
@@ -33,8 +33,8 @@ export class ConsoleReporter implements IReporter {
break;
}
- if (status != CheckStatus.Skipped) {
- console.log(`\t${statusLabel} : ${chalk.italic(url)}`)
+ if (check.status != CheckStatus.Skipped) {
+ console.log(`\t${statusLabel} : ${chalk.italic(check.url)} ${check.message ? `(${chalk.italic.grey(check.message)})` : ""}`)
}
}
@@ -66,7 +66,7 @@ export class ConsoleReporter implements IReporter {
break
}
- this.printCheck(check.status, check.url)
+ this.printCheck(check)
}
this.printTotals(oks, skipped, broken)
diff --git a/src/result.ts b/src/result.ts
index 3b0f24c495848a73f9e20f2e97c15bce83b6a5ad..ea1c25d4cc24037ba698d6a9f842c025d7794ea4 100644
--- a/src/result.ts
+++ b/src/result.ts
@@ -35,6 +35,7 @@ export class Result {
export class ResultItem {
public url = ""
public status = CheckStatus.OK
+ public duration = 0
public message?: string
}