From b26d96474bd63d01cde9345d82531388f7953f01 Mon Sep 17 00:00:00 2001
From: Dmytro Bogatov <dmytro@dbogatov.org>
Date: Wed, 24 Jun 2020 00:14:45 -0400
Subject: [PATCH] Improvements.
Add verbosity (with dots and x).
Add GET vs HEAD (retrying HEAD).
Add more prefixes to ignore.
Add JUnit reporter (test on GitLab).
Add option to ignore skipped URLs.
Check URL before running inspector.
Code up proper timeout.
Add <iframe src> match.
Suppress STDOUT in tests.
---
.gitignore | 2 +
package-lock.json | 140 +++++++++++++++++++++++++++++++++++++++++--
package.json | 6 +-
src/index.ts | 37 ++++++++++--
src/inspector.ts | 99 +++++++++++++++++++-----------
src/report.ts | 124 +++++++++++++++++++++++++++++++++++---
src/result.ts | 31 ++++++++--
test/extract-urls.ts | 5 ++
test/process-url.ts | 9 ++-
9 files changed, 395 insertions(+), 58 deletions(-)
diff --git a/.gitignore b/.gitignore
index ade49e1..0474a6e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,3 +3,5 @@ node_modules
.nyc_output
coverage
test-results.xml
+dump.lsif
+junit-report.xml
diff --git a/package-lock.json b/package-lock.json
index e668574..a498a5b 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -314,6 +314,46 @@
"integrity": "sha512-tsAQNx32a8CoFhjhijUIhI4kccIAgmGhy8LZMZgGfmXcpMbPRUqn5LWmgRttILi6yeGmBJd2xsPkFMs0PzgPCw==",
"dev": true
},
+ "@sinonjs/commons": {
+ "version": "1.8.0",
+ "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.8.0.tgz",
+ "integrity": "sha512-wEj54PfsZ5jGSwMX68G8ZXFawcSglQSXqCftWX3ec8MDUzQdHgcKvw97awHbY0efQEL5iKUOAmmVtoYgmrSG4Q==",
+ "requires": {
+ "type-detect": "4.0.8"
+ }
+ },
+ "@sinonjs/fake-timers": {
+ "version": "6.0.1",
+ "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-6.0.1.tgz",
+ "integrity": "sha512-MZPUxrmFubI36XS1DI3qmI0YdN1gks62JtFZvxR67ljjSNCeK6U08Zx4msEWOXuofgqUt6zPHSi1H9fbjR/NRA==",
+ "requires": {
+ "@sinonjs/commons": "^1.7.0"
+ }
+ },
+ "@sinonjs/formatio": {
+ "version": "5.0.1",
+ "resolved": "https://registry.npmjs.org/@sinonjs/formatio/-/formatio-5.0.1.tgz",
+ "integrity": "sha512-KaiQ5pBf1MpS09MuA0kp6KBQt2JUOQycqVG1NZXvzeaXe5LGFqAKueIS0bw4w0P9r7KuBSVdUk5QjXsUdu2CxQ==",
+ "requires": {
+ "@sinonjs/commons": "^1",
+ "@sinonjs/samsam": "^5.0.2"
+ }
+ },
+ "@sinonjs/samsam": {
+ "version": "5.0.3",
+ "resolved": "https://registry.npmjs.org/@sinonjs/samsam/-/samsam-5.0.3.tgz",
+ "integrity": "sha512-QucHkc2uMJ0pFGjJUDP3F9dq5dx8QIaqISl9QgwLOh6P9yv877uONPGXh/OH/0zmM3tW1JjuJltAZV2l7zU+uQ==",
+ "requires": {
+ "@sinonjs/commons": "^1.6.0",
+ "lodash.get": "^4.4.2",
+ "type-detect": "^4.0.8"
+ }
+ },
+ "@sinonjs/text-encoding": {
+ "version": "0.7.1",
+ "resolved": "https://registry.npmjs.org/@sinonjs/text-encoding/-/text-encoding-0.7.1.tgz",
+ "integrity": "sha512-+iTbntw2IZPb/anVDbypzfQa+ay64MW0Zo8aJ8gZPWMMK6/OubMVb6lUPMagqjOPnmtauXnFCACVl3O7ogjeqQ=="
+ },
"@types/chai": {
"version": "4.2.11",
"resolved": "https://registry.npmjs.org/@types/chai/-/chai-4.2.11.tgz",
@@ -336,6 +376,21 @@
"resolved": "https://registry.npmjs.org/@types/node/-/node-14.0.13.tgz",
"integrity": "sha512-rouEWBImiRaSJsVA+ITTFM6ZxibuAlTuNOCyxVbwreu6k6+ujs7DfnU9o+PShFhET78pMBl3eH+AGSI5eOTkPA=="
},
+ "@types/sinon": {
+ "version": "9.0.4",
+ "resolved": "https://registry.npmjs.org/@types/sinon/-/sinon-9.0.4.tgz",
+ "integrity": "sha512-sJmb32asJZY6Z2u09bl0G2wglSxDlROlAejCjsnor+LzBMz17gu8IU7vKC/vWDnv9zEq2wqADHVXFjf4eE8Gdw==",
+ "dev": true,
+ "requires": {
+ "@types/sinonjs__fake-timers": "*"
+ }
+ },
+ "@types/sinonjs__fake-timers": {
+ "version": "6.0.1",
+ "resolved": "https://registry.npmjs.org/@types/sinonjs__fake-timers/-/sinonjs__fake-timers-6.0.1.tgz",
+ "integrity": "sha512-yYezQwGWty8ziyYLdZjwxyMb0CZR49h8JALHGrxjQHWlqGgc8kLdHEgWrgL0uZ29DMvEVBDnHU2Wg36zKSIUtA==",
+ "dev": true
+ },
"aggregate-error": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.0.1.tgz",
@@ -691,8 +746,7 @@
"diff": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz",
- "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==",
- "dev": true
+ "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A=="
},
"dom-serializer": {
"version": "0.2.2",
@@ -1298,6 +1352,14 @@
"esprima": "^4.0.0"
}
},
+ "js2xmlparser": {
+ "version": "4.0.1",
+ "resolved": "https://registry.npmjs.org/js2xmlparser/-/js2xmlparser-4.0.1.tgz",
+ "integrity": "sha512-KrPTolcw6RocpYjdC7pL7v62e55q7qOMHvLX1UCLc5AAS8qeJ6nukarEJAF2KL2PZxlbGueEbINqZR2bDe/gUw==",
+ "requires": {
+ "xmlcreate": "^2.0.3"
+ }
+ },
"jsesc": {
"version": "2.5.2",
"resolved": "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz",
@@ -1313,6 +1375,11 @@
"minimist": "^1.2.5"
}
},
+ "just-extend": {
+ "version": "4.1.0",
+ "resolved": "https://registry.npmjs.org/just-extend/-/just-extend-4.1.0.tgz",
+ "integrity": "sha512-ApcjaOdVTJ7y4r08xI5wIqpvwS48Q0PBG4DJROcEkH1f8MdAiNFyFxz3xoL0LWAVwjrwPYZdVHHxhRHcx/uGLA=="
+ },
"locate-path": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz",
@@ -1334,6 +1401,11 @@
"integrity": "sha1-+wMJF/hqMTTlvJvsDWngAT3f7bI=",
"dev": true
},
+ "lodash.get": {
+ "version": "4.4.2",
+ "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz",
+ "integrity": "sha1-LRd/ZS+jHpObRDjVNBSZ36OCXpk="
+ },
"log-symbols": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-3.0.0.tgz",
@@ -1410,6 +1482,21 @@
"integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==",
"dev": true
},
+ "matcher": {
+ "version": "3.0.0",
+ "resolved": "https://registry.npmjs.org/matcher/-/matcher-3.0.0.tgz",
+ "integrity": "sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng==",
+ "requires": {
+ "escape-string-regexp": "^4.0.0"
+ },
+ "dependencies": {
+ "escape-string-regexp": {
+ "version": "4.0.0",
+ "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz",
+ "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA=="
+ }
+ }
+ },
"md5": {
"version": "2.2.1",
"resolved": "https://registry.npmjs.org/md5/-/md5-2.2.1.tgz",
@@ -1522,6 +1609,18 @@
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==",
"dev": true
},
+ "nise": {
+ "version": "4.0.3",
+ "resolved": "https://registry.npmjs.org/nise/-/nise-4.0.3.tgz",
+ "integrity": "sha512-EGlhjm7/4KvmmE6B/UFsKh7eHykRl9VH+au8dduHLCyWUO/hr7+N+WtTvDUwc9zHuM1IaIJs/0lQ6Ag1jDkQSg==",
+ "requires": {
+ "@sinonjs/commons": "^1.7.0",
+ "@sinonjs/fake-timers": "^6.0.0",
+ "@sinonjs/text-encoding": "^0.7.1",
+ "just-extend": "^4.0.2",
+ "path-to-regexp": "^1.7.0"
+ }
+ },
"node-preload": {
"version": "0.2.1",
"resolved": "https://registry.npmjs.org/node-preload/-/node-preload-0.2.1.tgz",
@@ -1765,6 +1864,21 @@
"integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==",
"dev": true
},
+ "path-to-regexp": {
+ "version": "1.8.0",
+ "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-1.8.0.tgz",
+ "integrity": "sha512-n43JRhlUKUAlibEJhPeir1ncUID16QnEjNpwzNdO3Lm4ywrBpBZ5oLD0I6br9evr1Y9JTqwRtAh7JLoOzAQdVA==",
+ "requires": {
+ "isarray": "0.0.1"
+ },
+ "dependencies": {
+ "isarray": {
+ "version": "0.0.1",
+ "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz",
+ "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8="
+ }
+ }
+ },
"pathval": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/pathval/-/pathval-1.1.0.tgz",
@@ -1907,6 +2021,20 @@
"integrity": "sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA==",
"dev": true
},
+ "sinon": {
+ "version": "9.0.2",
+ "resolved": "https://registry.npmjs.org/sinon/-/sinon-9.0.2.tgz",
+ "integrity": "sha512-0uF8Q/QHkizNUmbK3LRFqx5cpTttEVXudywY9Uwzy8bTfZUhljZ7ARzSxnRHWYWtVTeh4Cw+tTb3iU21FQVO9A==",
+ "requires": {
+ "@sinonjs/commons": "^1.7.2",
+ "@sinonjs/fake-timers": "^6.0.1",
+ "@sinonjs/formatio": "^5.0.1",
+ "@sinonjs/samsam": "^5.0.3",
+ "diff": "^4.0.2",
+ "nise": "^4.0.1",
+ "supports-color": "^7.1.0"
+ }
+ },
"source-map": {
"version": "0.5.7",
"resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz",
@@ -2052,8 +2180,7 @@
"type-detect": {
"version": "4.0.8",
"resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz",
- "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==",
- "dev": true
+ "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g=="
},
"type-fest": {
"version": "0.8.1",
@@ -2199,6 +2326,11 @@
"integrity": "sha1-eLpyAgApxbyHuKgaPPzXS0ovweU=",
"dev": true
},
+ "xmlcreate": {
+ "version": "2.0.3",
+ "resolved": "https://registry.npmjs.org/xmlcreate/-/xmlcreate-2.0.3.tgz",
+ "integrity": "sha512-HgS+X6zAztGa9zIK3Y3LXuJes33Lz9x+YyTxgrkIdabu2vqcGOWwdfCpf1hWLRrd553wd4QCDf6BBO6FfdsRiQ=="
+ },
"y18n": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.0.tgz",
diff --git a/package.json b/package.json
index 766b510..aff1e5f 100644
--- a/package.json
+++ b/package.json
@@ -34,11 +34,15 @@
"axios": "^0.19.2",
"chalk": "^4.1.0",
"commander": "^5.1.0",
- "htmlparser2": "^4.1.0"
+ "htmlparser2": "^4.1.0",
+ "js2xmlparser": "^4.0.1",
+ "matcher": "^3.0.0",
+ "sinon": "^9.0.2"
},
"devDependencies": {
"@types/chai": "^4.2.11",
"@types/mocha": "^7.0.2",
+ "@types/sinon": "^9.0.4",
"chai": "^4.2.0",
"mocha": "^8.0.1",
"mocha-junit-reporter": "^2.0.0",
diff --git a/src/index.ts b/src/index.ts
index dcca898..08d3ec1 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -1,7 +1,7 @@
import commander from "commander"
import chalk from "chalk"
import { Inspector, URLsMatchingSet, Config } from "./inspector"
-import { ConsoleReporter } from "./report"
+import { ConsoleReporter, JUnitReporter } from "./report"
commander
.version("0.1.0") // TODO automatically
@@ -11,22 +11,47 @@ commander
.command("inspect <url>")
.description("Check links in the given URL")
.option("-r, --recursive", "recursively check all links in all URLs within supplied host", false)
+ .option("-v, --verbose", "log progress of checking URLs", false)
+ .option("-g, --get", "use GET request instead of HEAD", false)
.option("-t, --timeout <number>", "timeout in ms after which the link will be considered broken", (value: string, _) => parseInt(value), 2000)
- .option("--ignore-prefixes <coma-separated-strings>", "prefix(es) to ignore (without ':'), like mailto: and tel:", (value: string, _) => value.split(","), ["mailto", "tel"])
+ .option("--ignore-prefixes <coma-separated-strings>", "prefix(es) to ignore (without ':'), like mailto: and tel:", (value: string, _) => value.split(","), ["javascript", "data", "mailto", "sms", "tel", "geo"])
.option("--accept-codes <coma-separated-numbers>", "HTTP response code(s) (beyond 200-299) to accept, like 999 for linkedin", (value: string, _) => value.split(",").map(code => parseInt(code)), [999])
+ .option("--reporters <coma-separated-strings>", "Reporters to use in processing the results (junit, console)", (value: string, _) => value.split(","), ["console"])
+ .option("--ignore-skipped", "Do not report skipped URLs", false)
.option("-s, --skip <globs>", "URLs to skip defined by globs, like '*linkedin*'", (value: string, previous: string[]) => previous.concat([value]), [])
.action(async (url: string, inspectObj) => {
+ try {
+ new URL(url)
+ } catch (e) {
+ console.error(chalk.red(`${url} does not look like valid URL (forgot http(s)?)`))
+ process.exit(1)
+ }
+
let inspector = new Inspector(new URLsMatchingSet(), {
acceptedCodes: inspectObj.acceptCodes as number[],
timeout: parseInt(inspectObj.timeout as string),
ignoredPrefixes: inspectObj.ignorePrefixes as string[],
- skipURLs: inspectObj.skip as string[]
+ skipURLs: inspectObj.skip as string[],
+ verbose: inspectObj.verbose as boolean,
+ get: inspectObj.get as boolean,
+ ignoreSkipped: inspectObj.ignoreSkipped as boolean
})
- let result = await inspector.processURL(new URL(url), inspectObj.recursive)
- let success = result.report(new ConsoleReporter())
- process.exit(success ? 0 : 1)
+ let result = await inspector.processURL(new URL(url), inspectObj.recursive as boolean)
+
+ for (const reporter of inspectObj.reporters as string[]) {
+ switch (reporter) {
+ case "junit":
+ result.report(new JUnitReporter())
+ break
+ case "console":
+ result.report(new ConsoleReporter())
+ break
+ }
+ }
+
+ process.exit(result.success() ? 0 : 1)
})
if (!process.argv.slice(2).length) {
diff --git a/src/inspector.ts b/src/inspector.ts
index 1e6e1af..313f5a6 100644
--- a/src/inspector.ts
+++ b/src/inspector.ts
@@ -1,7 +1,7 @@
import * as parser from "htmlparser2"
import axios, { AxiosError } from "axios"
import { Result, CheckStatus } from "./result";
-import { performance } from "perf_hooks"
+import { isMatch } from "matcher"
export class Inspector {
@@ -10,64 +10,83 @@ export class Inspector {
private readonly config: Config
) { }
+ async timeout<T>(timeoutMs: number, promise: () => Promise<T>, failureMessage: string = "timeout"): Promise<T> {
+ let timeoutHandle: NodeJS.Timeout | undefined
+ const timeoutPromise = new Promise<never>((resolve, reject) => {
+ timeoutHandle = setTimeout(() => reject(new Error(failureMessage)), timeoutMs)
+ })
+
+ const result = await Promise.race([
+ promise(),
+ timeoutPromise
+ ]);
+ clearTimeout(timeoutHandle!);
+ return result;
+ }
+
async processURL(originalUrl: URL, recursive: boolean): Promise<Result> {
- let result = new Result();
- let urlsToCheck: [string, string?][] = [[originalUrl.href, undefined]]
+ let result = new Result(this.config.ignoreSkipped);
+ // [url, GET, parent?]
+ let urlsToCheck: [string, boolean, string?][] = [[originalUrl.href, true, undefined]]
- let processingRoutine = async (url: string, parent?: string) => {
+ let processingRoutine = async (url: string, useGet: boolean, parent?: string) => {
try {
- url = parent ? new URL(url, parent).href : url
-
- if (result.isChecked(url) || this.config.ignoredPrefixes.some(ext => url.startsWith(ext + ":"))) {
- result.add({ url: url, status: CheckStatus.Skipped, duration: 0 }, parent)
+ url = parent ? new URL(url, parent).href : new URL(url).href
+ if (url.includes("#")) {
+ url = url.split("#")[0]
+ }
+ let shouldParse = url == originalUrl.href || (recursive && originalUrl.origin == new URL(url).origin)
+
+ if (
+ result.isChecked(url) ||
+ this.config.ignoredPrefixes.some(ext => url.startsWith(ext + ":")) ||
+ this.config.skipURLs.some(glob => url.includes(glob) || isMatch(url, glob))
+ ) {
+ result.add({ url: url, status: CheckStatus.Skipped }, parent)
} else {
+ let urlToCheck = parent ? new URL(url, parent).href : url
const instance = axios.create()
- instance.interceptors.request.use(config => {
- config.headers["request-startTime"] = performance.now()
- return config
- })
- instance.interceptors.response.use((response) => {
- const start = response.config.headers["request-startTime"]
- const end = performance.now()
- response.headers["request-duration"] = end - start
- return response
- })
- const response = await instance.get(parent ? new URL(url, parent).href : url, { timeout: this.config.timeout })
- const duration = response.headers["request-duration"]
+ const response = useGet || shouldParse ?
+ await this.timeout(this.config.timeout, () => instance.get(urlToCheck)) :
+ await this.timeout(this.config.timeout, () => instance.head(urlToCheck))
let html = response.data as string
- if (url == originalUrl.href || (recursive && originalUrl.origin == new URL(url).origin)) {
+ if (shouldParse) {
let discoveredURLs = this.extractURLs(html)
for (const discovered of discoveredURLs) {
- urlsToCheck.push([discovered, url])
+ urlsToCheck.push([discovered, this.config.get, url])
}
}
- result.add({ url: url, status: CheckStatus.OK, duration: duration }, parent)
+ result.add({ url: url, status: CheckStatus.OK }, parent)
}
} catch (exception) {
const error: AxiosError = exception;
- if ((exception.message as string).includes("timeout")) {
- result.add({ url: url, status: CheckStatus.Timeout, duration: this.config.timeout }, parent)
- } else if (!error.response) {
- result.add({ url: url, status: CheckStatus.GenericError, duration: 0 }, parent)
+ // if HEAD was used, retry with GET
+ if (!useGet) {
+ urlsToCheck.push([url, true, parent])
} else {
- const duration = performance.now() - error.response.config.headers["request-startTime"]
-
- if (this.config.acceptedCodes.some(code => code == error.response?.status)) {
- result.add({ url: url, status: CheckStatus.OK, duration: duration }, parent)
+ if ((exception.message as string).includes("timeout")) {
+ result.add({ url: url, status: CheckStatus.Timeout }, parent)
+ } else if (!error.response) {
+ result.add({ url: url, status: CheckStatus.GenericError }, parent)
} else {
- result.add({ url: url, status: CheckStatus.NonSuccessCode, message: `${error.response.status}`, duration: duration }, parent)
+ if (this.config.acceptedCodes.some(code => code == error.response?.status)) {
+ result.add({ url: url, status: CheckStatus.OK }, parent)
+ } else {
+ result.add({ url: url, status: CheckStatus.NonSuccessCode, message: `${error.response.status}` }, parent)
+ }
}
}
+
}
}
@@ -76,15 +95,16 @@ export class Inspector {
while (urlsToCheck.length > 0) {
- let [url, parent] = urlsToCheck.pop()!
+ let [url, useGet, parent] = urlsToCheck.pop()!
- promises.push(processingRoutine(url, parent))
+ promises.push(processingRoutine(url, useGet, parent))
if (urlsToCheck.length == 0) {
await Promise.all(promises)
}
}
+ console.log()
return result
}
@@ -117,13 +137,17 @@ export class Config {
timeout: number = 2000
ignoredPrefixes: string[] = ["mailto", "tel"]
skipURLs: string[] = []
+ verbose: boolean = false
+ get: boolean = false
+ ignoreSkipped: boolean = false
}
export enum URLMatchingRule {
AHRef = "<a href>",
ScriptSrc = "<script src>",
LinkHref = "<link href>",
- ImgSrc = "<img src>"
+ ImgSrc = "<img src>",
+ IFrameSrc = "<iframe src>"
}
export class URLsMatchingSet {
@@ -157,6 +181,11 @@ export class URLsMatchingSet {
return attributes.src
}
break;
+ case URLMatchingRule.IFrameSrc:
+ if (name === "iframe" && "src" in attributes) {
+ return attributes.src
+ }
+ break;
default:
throw new Error(`unknown rule: ${rule}`);
}
diff --git a/src/report.ts b/src/report.ts
index 5f8da3e..b95bb6f 100644
--- a/src/report.ts
+++ b/src/report.ts
@@ -1,11 +1,123 @@
-import { ResultItem, CheckStatus } from "./result";
-import chalk from "chalk";
+import { ResultItem, CheckStatus } from "./result"
+import chalk from "chalk"
+import { parse } from "js2xmlparser"
+
+import fs from "fs";
export interface IReporter {
- process(pages: Map<string, ResultItem[]>): boolean
+ process(pages: Map<string, ResultItem[]>): void
}
-// ConsoleReporter
+/**
+ * testsuite = [
+ * {
+ * "@": {
+ * name: "parent URL",
+ * tests: totalTests
+ * failures: broken,
+ * skipped: skipped
+ * }
+ * "testcase" = [
+ * {
+ * "@": {
+ * name: URL,
+ * },
+ * error?: {
+ * "@": {
+ * message: "error message"
+ * }
+ * },
+ * skipped?: {}
+ * }
+ * ]
+ * }
+ * ]
+ */
+export class JUnitReporter implements IReporter {
+
+ constructor(readonly toFile: boolean = true) { }
+
+ process(pages: Map<string, ResultItem[]>): void {
+
+ let junitObject: any[] = []
+
+ for (const page of pages) {
+
+ let testsuite: any = {
+ testcase: []
+ }
+
+ let skipped = 0
+ let oks = 0
+ let broken = 0
+
+ for (const check of page[1]) {
+
+ let testcase: any = {
+ "@": {
+ name: check.url,
+ classname: page[0],
+ time: "0.0000"
+ }
+ }
+
+ switch (check.status) {
+ case CheckStatus.NonSuccessCode:
+ testcase["failure"] = {
+ "@": {
+ message: check.message
+ }
+ }
+ broken++
+ break
+ case CheckStatus.GenericError:
+ testcase["failure"] = {
+ "@": {
+ message: "Unknown error"
+ }
+ }
+ broken++
+ break
+ case CheckStatus.Timeout:
+ testcase["failure"] = {
+ "@": {
+ message: "Timeout"
+ }
+ }
+ broken++
+ break
+ case CheckStatus.Skipped:
+ testcase["skipped"] = {}
+ skipped++
+ break
+ case CheckStatus.OK:
+ oks++
+ break
+ }
+
+ testsuite.testcase.push(testcase)
+ }
+
+ testsuite["@"] = {
+ name: page[0],
+ tests: oks + skipped + broken,
+ failures: broken,
+ skipped: skipped,
+ time: "0.0000"
+ }
+
+ junitObject.push(testsuite)
+ }
+
+ let junitXml = parse("testsuites", { testsuite: junitObject })
+ if (this.toFile) {
+ fs.writeFileSync("junit-report.xml", junitXml)
+ } else {
+ console.log(junitXml)
+ }
+ }
+
+}
export class ConsoleReporter implements IReporter {
@@ -38,7 +150,7 @@ export class ConsoleReporter implements IReporter {
}
}
- process(pages: Map<string, ResultItem[]>): boolean {
+ process(pages: Map<string, ResultItem[]>) {
let allSkipped = 0
let allOks = 0
@@ -75,8 +187,6 @@ export class ConsoleReporter implements IReporter {
allBroken += broken
}
this.printTotals(allOks, allSkipped, allBroken, false)
-
- return allBroken == 0
}
}
diff --git a/src/result.ts b/src/result.ts
index ea1c25d..1123e5b 100644
--- a/src/result.ts
+++ b/src/result.ts
@@ -3,9 +3,21 @@ import { IReporter } from "./report"
export class Result {
private pages = new Map<string, ResultItem[]>()
private checkedUrls = new Set<string>()
+ private addedCount = 0
+ private atLeastOneBroken = false
+
+ constructor(readonly ignoreSkipped: boolean) { }
public add(completedCheck: ResultItem, parent: string = "original request") {
- // console.log(`${completedCheck.url} : ${completedCheck.status} ${completedCheck.message ? completedCheck.message : ""}`) // TODO
+ if (completedCheck.status == CheckStatus.Skipped && this.ignoreSkipped) {
+ return
+ }
+
+ if (this.addedCount > 0 && this.addedCount % 80 == 0) {
+ process.stdout.write("\n")
+ }
+ process.stdout.write(completedCheck.status == CheckStatus.OK || completedCheck.status == CheckStatus.Skipped ? "." : "x")
+ this.addedCount++
if (this.pages.has(parent)) {
this.pages.get(parent)?.push(completedCheck)
@@ -13,6 +25,14 @@ export class Result {
this.pages.set(parent, [completedCheck])
}
this.checkedUrls.add(completedCheck.url)
+
+ if (
+ completedCheck.status == CheckStatus.GenericError ||
+ completedCheck.status == CheckStatus.Timeout ||
+ completedCheck.status == CheckStatus.NonSuccessCode
+ ) {
+ this.atLeastOneBroken = true
+ }
}
public isChecked(url: string): boolean {
@@ -27,15 +47,18 @@ export class Result {
return count
}
- public report<ReporterT extends IReporter>(reporter: ReporterT): boolean {
- return reporter.process(this.pages)
+ public report<ReporterT extends IReporter>(reporter: ReporterT): void {
+ reporter.process(this.pages)
+ }
+
+ public success() {
+ return !this.atLeastOneBroken
}
}
export class ResultItem {
public url = ""
public status = CheckStatus.OK
- public duration = 0
public message?: string
}
diff --git a/test/extract-urls.ts b/test/extract-urls.ts
index d56611a..17d6f11 100644
--- a/test/extract-urls.ts
+++ b/test/extract-urls.ts
@@ -26,6 +26,11 @@ describe("extractURLs", () => {
expect(result).to.eql(new Set([url]))
});
+ it("works for <iframe src=...>", () => {
+ const result = new Inspector(new URLsMatchingSet(URLMatchingRule.IFrameSrc), new Config()).extractURLs(`<html><iframe src="${url}">Text</iframe></html>`)
+ expect(result).to.eql(new Set([url]))
+ });
+
it("works for many rules", () => {
const result = new Inspector(new URLsMatchingSet(), new Config())
.extractURLs(
diff --git a/test/process-url.ts b/test/process-url.ts
index 0f1f852..c397ff4 100644
--- a/test/process-url.ts
+++ b/test/process-url.ts
@@ -1,7 +1,8 @@
import { Inspector, URLsMatchingSet, URLMatchingRule, Config } from "../src/inspector"
import { expect, assert } from "chai";
import "mocha";
-import { ConsoleReporter } from "../src/report";
+import { ConsoleReporter, JUnitReporter } from "../src/report";
+import sinon from "sinon"
describe("processURL", function () {
@@ -9,6 +10,11 @@ describe("processURL", function () {
const validURL = new URL("https://dbogatov.org")
+ before(function () {
+ sinon.stub(console, "log")
+ sinon.stub(process.stdout, "write")
+ });
+
it("processes non-recursive", async () => {
const result = await new Inspector(new URLsMatchingSet(), new Config()).processURL(validURL, false)
@@ -21,6 +27,7 @@ describe("processURL", function () {
const result = await new Inspector(new URLsMatchingSet(), new Config()).processURL(validURL, true)
result.report(new ConsoleReporter())
+ result.report(new JUnitReporter(false))
// assert(result.length == 1)
// assert(result[0].url === validURL.href)
--
GitLab