Skip to content
Snippets Groups Projects
Verified Commit 85ac8d3d authored by Dmytro Bogatov's avatar Dmytro Bogatov :two_hearts:
Browse files

Work on CLI.

Add timeout, accepted-code, ignore-prefixes and skip.
Include processing of these options.
Track request time (poorly).
parent 40a7b69e
No related branches found
No related tags found
No related merge requests found
Pipeline #7064 passed
......@@ -25,7 +25,14 @@
"program": "${workspaceFolder}/dist/index.js",
"args": [
"inspect",
"https://dbogatov.org"
"https://dbogatov.org",
"-r",
"-t",
"100",
"--ignore-prefixes",
"m,l",
"--accept-codes",
"444,555"
],
"console": "integratedTerminal",
"internalConsoleOptions": "neverOpen",
......
......@@ -11,9 +11,18 @@ commander
.command("inspect <url>")
.description("Check links in the given URL")
.option("-r, --recursive", "recursively check all links in all URLs within supplied host", false)
.option("-t, --timeout <number>", "timeout in ms after which the link will be considered broken", (value: string, _) => parseInt(value), 2000)
.option("--ignore-prefixes <coma-separated-strings>", "prefix(es) to ignore (without ':'), like mailto: and tel:", (value: string, _) => value.split(","), ["mailto", "tel"])
.option("--accept-codes <coma-separated-numbers>", "HTTP response code(s) (beyond 200-299) to accept, like 999 for linkedin", (value: string, _) => value.split(",").map(code => parseInt(code)), [999])
.option("-s, --skip <globs>", "URLs to skip defined by globs, like '*linkedin*'", (value: string, previous: string[]) => previous.concat([value]), [])
.action(async (url: string, inspectObj) => {
let inspector = new Inspector(new URLsMatchingSet(), new Config())
let inspector = new Inspector(new URLsMatchingSet(), {
acceptedCodes: inspectObj.acceptCodes as number[],
timeout: parseInt(inspectObj.timeout as string),
ignoredPrefixes: inspectObj.ignorePrefixes as string[],
skipURLs: inspectObj.skip as string[]
})
let result = await inspector.processURL(new URL(url), inspectObj.recursive)
let success = result.report(new ConsoleReporter())
......
import * as parser from "htmlparser2"
import axios, { AxiosError } from "axios"
import { Result, CheckStatus } from "./result";
import { performance } from "perf_hooks"
export class Inspector {
......@@ -19,14 +20,27 @@ export class Inspector {
try {
url = parent ? new URL(url, parent).href : url
if (result.isChecked(url) || this.config.ignoredExtensions.some(ext => url.startsWith(ext + ":"))) {
result.add({ url: url, status: CheckStatus.Skipped }, parent)
if (result.isChecked(url) || this.config.ignoredPrefixes.some(ext => url.startsWith(ext + ":"))) {
result.add({ url: url, status: CheckStatus.Skipped, duration: 0 }, parent)
} else {
const response = await axios.get(parent ? new URL(url, parent).href : url, { timeout: this.config.timeout })
const instance = axios.create()
instance.interceptors.request.use(config => {
config.headers["request-startTime"] = performance.now()
return config
})
instance.interceptors.response.use((response) => {
const start = response.config.headers["request-startTime"]
const end = performance.now()
response.headers["request-duration"] = end - start
return response
})
const response = await instance.get(parent ? new URL(url, parent).href : url, { timeout: this.config.timeout })
const duration = response.headers["request-duration"]
let html = response.data as string
if (url == originalUrl.href || (recursive && originalUrl.host == new URL(url).host)) {
if (url == originalUrl.href || (recursive && originalUrl.origin == new URL(url).origin)) {
let discoveredURLs = this.extractURLs(html)
......@@ -35,20 +49,23 @@ export class Inspector {
}
}
result.add({ url: url, status: CheckStatus.OK }, parent)
result.add({ url: url, status: CheckStatus.OK, duration: duration }, parent)
}
} catch (exception) {
const error: AxiosError = exception;
if (!error.response) {
result.add({ url: url, status: CheckStatus.GenericError }, parent)
if ((exception.message as string).includes("timeout")) {
result.add({ url: url, status: CheckStatus.Timeout, duration: this.config.timeout }, parent)
} else if (!error.response) {
result.add({ url: url, status: CheckStatus.GenericError, duration: 0 }, parent)
} else {
const duration = performance.now() - error.response.config.headers["request-startTime"]
if (this.config.acceptedCodes.some(code => code == error.response?.status)) {
result.add({ url: url, status: CheckStatus.OK }, parent)
result.add({ url: url, status: CheckStatus.OK, duration: duration }, parent)
} else {
result.add({ url: url, status: CheckStatus.NonSuccessCode, message: `${error.response.status}` }, parent)
result.add({ url: url, status: CheckStatus.NonSuccessCode, message: `${error.response.status}`, duration: duration }, parent)
}
}
}
......@@ -96,9 +113,10 @@ export class Inspector {
export class Config {
public acceptedCodes: number[] = [999]
public timeout: number = 2000
public ignoredExtensions: string[] = ["mailto", "tel"]
acceptedCodes: number[] = [999]
timeout: number = 2000
ignoredPrefixes: string[] = ["mailto", "tel"]
skipURLs: string[] = []
}
export enum URLMatchingRule {
......
......@@ -13,11 +13,11 @@ export class ConsoleReporter implements IReporter {
console.log(`${indent ? "\t" : ""}${chalk.green(`OK: ${oks}`)}, ${chalk.grey(`skipped: ${skipped}`)}, ${chalk.red(`broken: ${broken}`)}`)
}
private printCheck(status: CheckStatus, url: string) {
private printCheck(check: ResultItem) {
let statusLabel: string
const labelWidth = 7
switch (status) {
switch (check.status) {
case CheckStatus.OK:
statusLabel = chalk.green("OK".padEnd(labelWidth))
break;
......@@ -33,8 +33,8 @@ export class ConsoleReporter implements IReporter {
break;
}
if (status != CheckStatus.Skipped) {
console.log(`\t${statusLabel} : ${chalk.italic(url)}`)
if (check.status != CheckStatus.Skipped) {
console.log(`\t${statusLabel} : ${chalk.italic(check.url)} ${check.message ? `(${chalk.italic.grey(check.message)})` : ""}`)
}
}
......@@ -66,7 +66,7 @@ export class ConsoleReporter implements IReporter {
break
}
this.printCheck(check.status, check.url)
this.printCheck(check)
}
this.printTotals(oks, skipped, broken)
......
......@@ -35,6 +35,7 @@ export class Result {
export class ResultItem {
public url = ""
public status = CheckStatus.OK
public duration = 0
public message?: string
}
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment