Skip to content
Snippets Groups Projects
Verified Commit 8ac06b7a authored by Dmytro Bogatov's avatar Dmytro Bogatov :two_hearts:
Browse files

Add ESLint.

parent ed0ff68d
Branches
No related tags found
No related merge requests found
Pipeline #7439 failed
env:
browser: true
es2020: true
extends:
- "eslint:recommended"
- "plugin:@typescript-eslint/recommended"
parser: "@typescript-eslint/parser"
parserOptions:
ecmaVersion: 11
sourceType: module
plugins:
- "@typescript-eslint"
rules:
indent:
- error
- tab
- SwitchCase: 1
linebreak-style:
- error
- unix
quotes:
- error
- double
semi:
- error
- never
"@typescript-eslint/no-unused-vars":
- error
- argsIgnorePattern: "^_"
......@@ -2,6 +2,15 @@ stages:
- test
- release
lint:
image: dbogatov/docker-sources:node--14.4-alpine3.12
stage: test
script:
- npm -g install eslint
- eslint ./{src,test}/**/*.ts -c .eslintrc.yml
tags:
- docker
test:
image: dbogatov/docker-sources:node--14.4-alpine3.12
stage: test
......
......
This diff is collapsed.
{
"name": "broken-links-inspector",
"version": "1.1.1",
"version": "1.1.2",
"description": "Extract and recursively check all URLs reporting broken ones",
"main": "dist/inspector.js",
"types": "dist/inspector.d.ts",
......@@ -59,7 +59,10 @@
"@types/chai": "^4.2.11",
"@types/mocha": "^7.0.2",
"@types/sinon": "^9.0.4",
"@typescript-eslint/eslint-plugin": "^3.4.0",
"@typescript-eslint/parser": "^3.4.0",
"chai": "^4.2.0",
"eslint": "^7.3.1",
"mocha": "^8.0.1",
"mocha-junit-reporter": "^2.0.0",
"nyc": "^15.1.0",
......
......
......@@ -6,7 +6,7 @@ import { Inspector, URLsMatchingSet } from "./inspector"
import { ConsoleReporter, JUnitReporter } from "./report"
commander
.version("1.1.1")
.version("1.1.2")
.description("Extract and recursively check all URLs reporting broken ones")
commander
......@@ -30,7 +30,7 @@ commander
process.exit(1)
}
let inspector = new Inspector(new URLsMatchingSet(), {
const inspector = new Inspector(new URLsMatchingSet(), {
acceptedCodes: inspectObj.acceptCodes as number[],
timeout: parseInt(inspectObj.timeout as string),
ignoredPrefixes: inspectObj.ignorePrefixes as string[],
......@@ -41,7 +41,7 @@ commander
disablePrint: false
})
let result = await inspector.processURL(new URL(url), inspectObj.recursive as boolean)
const result = await inspector.processURL(new URL(url), inspectObj.recursive as boolean)
for (const reporter of inspectObj.reporters as string[]) {
switch (reporter) {
......
......
import * as parser from "htmlparser2"
import axios, { AxiosError } from "axios"
import { Result, CheckStatus } from "./result";
import { Result, CheckStatus } from "./result"
import { isMatch } from "matcher"
export interface IHttpClient {
......@@ -21,7 +21,7 @@ export class AxiosHttpClient implements IHttpClient {
readonly acceptedCodes: number[]
) { }
private async timeoutWrapper<T>(timeoutMs: number, promise: () => Promise<T>, failureMessage: string = "timeout"): Promise<T> {
private async timeoutWrapper<T>(timeoutMs: number, promise: () => Promise<T>, failureMessage = "timeout"): Promise<T> {
let timeoutHandle: NodeJS.Timeout | undefined
const timeoutPromise = new Promise<never>((_, reject) => {
timeoutHandle = setTimeout(() => reject(new Error(failureMessage)), timeoutMs)
......@@ -30,9 +30,11 @@ export class AxiosHttpClient implements IHttpClient {
const result = await Promise.race([
promise(),
timeoutPromise
]);
clearTimeout(timeoutHandle!);
return result;
])
if (timeoutHandle) {
clearTimeout(timeoutHandle)
}
return result
}
async request(get: boolean, url: string): Promise<string> {
......@@ -43,13 +45,14 @@ export class AxiosHttpClient implements IHttpClient {
return (await this.timeoutWrapper(this.timeout, () => get ? instance.get(url) : instance.head(url))).data as string
} catch (exception) {
const error: AxiosError = exception;
const error: AxiosError = exception
if ((exception.message as string).includes("timeout")) {
throw new HttpClientFailure(true, -1)
} else if (!error.response) {
throw new HttpClientFailure(false, -1)
} else {
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
if (this.acceptedCodes.some(code => code == error.response!.status)) {
return ""
} else {
......@@ -70,11 +73,11 @@ export class Inspector {
async processURL(originalUrl: URL, recursive: boolean): Promise<Result> {
let result = new Result(this.config.ignoreSkipped, this.config.disablePrint);
const result = new Result(this.config.ignoreSkipped, this.config.disablePrint)
// [url, GET, parent?]
let urlsToCheck: [string, boolean, string?][] = [[originalUrl.href, true, undefined]]
const urlsToCheck: [string, boolean, string?][] = [[originalUrl.href, true, undefined]]
let processingRoutine = async (url: string, useGet: boolean, parent?: string) => {
const processingRoutine = async (url: string, useGet: boolean, parent?: string) => {
try {
try {
......@@ -85,7 +88,7 @@ export class Inspector {
if (url.includes("#")) {
url = url.split("#")[0]
}
let shouldParse = url == originalUrl.href || (recursive && originalUrl.origin == new URL(url).origin)
const shouldParse = url == originalUrl.href || (recursive && originalUrl.origin == new URL(url).origin)
if (
result.isChecked(url) ||
......@@ -94,13 +97,13 @@ export class Inspector {
) {
result.add({ url: url, status: CheckStatus.Skipped }, parent)
} else {
let urlToCheck = parent ? new URL(url, parent).href : url
const urlToCheck = parent ? new URL(url, parent).href : url
let html = await this.httpClient.request(useGet || shouldParse, urlToCheck)
const html = await this.httpClient.request(useGet || shouldParse, urlToCheck)
if (shouldParse) {
let discoveredURLs = this.extractURLs(html)
const discoveredURLs = this.extractURLs(html)
for (const discovered of discoveredURLs) {
urlsToCheck.push([discovered, this.config.get, url])
......@@ -111,7 +114,7 @@ export class Inspector {
}
} catch (exception) {
const error: HttpClientFailure = exception;
const error: HttpClientFailure = exception
// if HEAD was used, retry with GET
if (!useGet) {
......@@ -128,11 +131,12 @@ export class Inspector {
}
}
let promises: Promise<void>[] = []
const promises: Promise<void>[] = []
while (urlsToCheck.length > 0) {
let [url, useGet, parent] = urlsToCheck.pop()!
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
const [url, useGet, parent] = urlsToCheck.pop()!
promises.push(processingRoutine(url, useGet, parent))
......@@ -147,13 +151,13 @@ export class Inspector {
extractURLs(html: string): Set<string> {
let urls = new Set<string>();
let matcher = this.matcher
const urls = new Set<string>()
const matcher = this.matcher
let parserInstance = new parser.Parser(
const parserInstance = new parser.Parser(
{
onopentag(name, attributes) {
const match = matcher.match(name, attributes);
const match = matcher.match(name, attributes)
if (match && match !== "" && !match.startsWith("#")) {
urls.add(match)
}
......@@ -171,13 +175,13 @@ export class Inspector {
export class Config {
acceptedCodes: number[] = [999]
timeout: number = 2000
timeout = 2000
ignoredPrefixes: string[] = ["mailto", "tel"]
skipURLs: string[] = []
verbose: boolean = false
get: boolean = false
ignoreSkipped: boolean = false
disablePrint: boolean = false
verbose = false
get = false
ignoreSkipped = false
disablePrint = false
}
export enum URLMatchingRule {
......@@ -192,7 +196,7 @@ export class URLsMatchingSet {
private rules: URLMatchingRule[]
constructor(...rules: URLMatchingRule[]) {
this.rules = rules.length > 0 ? rules : Object.values(URLMatchingRule);
this.rules = rules.length > 0 ? rules : Object.values(URLMatchingRule)
}
public match(name: string, attributes: { [s: string]: string }): string | undefined {
......@@ -203,29 +207,29 @@ export class URLsMatchingSet {
if (name === "a" && "href" in attributes) {
return attributes.href
}
break;
break
case URLMatchingRule.ScriptSrc:
if (name === "script" && "src" in attributes) {
return attributes.src
}
break;
break
case URLMatchingRule.LinkHref:
if (name === "link" && "href" in attributes) {
return attributes.href
}
break;
break
case URLMatchingRule.ImgSrc:
if (name === "img" && "src" in attributes) {
return attributes.src
}
break;
break
case URLMatchingRule.IFrameSrc:
if (name === "iframe" && "src" in attributes) {
return attributes.src
}
break;
break
default:
throw new Error(`unknown rule: ${rule}`);
throw new Error(`unknown rule: ${rule}`)
}
}
......
......
import { ResultItem, CheckStatus } from "./result"
import chalk from "chalk"
import { parse } from "js2xmlparser"
import fs from "fs";
import fs from "fs"
export interface IReporter {
process(pages: Map<string, ResultItem[]>): any
process(pages: Map<string, ResultItem[]>): unknown
}
/**
......@@ -38,11 +38,35 @@ export class JUnitReporter implements IReporter {
process(pages: Map<string, ResultItem[]>): void {
let junitObject: any[] = []
type TestCase = {
"@": {
name: string,
classname: string,
time: string,
},
failure?: {
"@": {
message?: string
}
},
skipped?: unknown
}
type TestSuite = {
testcase: TestCase[],
"@"?: {
name: string,
tests: number,
failures: number,
skipped: number,
time: string
}
}
const junitObject: TestSuite[] = []
for (const page of pages) {
let testsuite: any = {
const testsuite: TestSuite = {
testcase: []
}
......@@ -52,7 +76,7 @@ export class JUnitReporter implements IReporter {
for (const check of page[1]) {
let testcase: any = {
const testcase: TestCase = {
"@": {
name: check.url,
classname: page[0],
......@@ -108,7 +132,7 @@ export class JUnitReporter implements IReporter {
junitObject.push(testsuite)
}
let junitXml = parse("testsuites", { testsuite: junitObject })
const junitXml = parse("testsuites", { testsuite: junitObject })
if (this.toFile) {
fs.writeFileSync("junit-report.xml", junitXml)
} else {
......@@ -120,7 +144,7 @@ export class JUnitReporter implements IReporter {
export class ConsoleReporter implements IReporter {
private printTotals(oks: number, skipped: number, broken: number, indent: boolean = true) {
private printTotals(oks: number, skipped: number, broken: number, indent = true) {
console.log(`${indent ? "\t" : ""}${chalk.green(`OK: ${oks}`)}, ${chalk.grey(`skipped: ${skipped}`)}, ${chalk.red(`broken: ${broken}`)}`)
}
......@@ -131,17 +155,17 @@ export class ConsoleReporter implements IReporter {
switch (check.status) {
case CheckStatus.OK:
statusLabel = chalk.green("OK".padEnd(labelWidth))
break;
break
case CheckStatus.Skipped:
statusLabel = chalk.gray("SKIP".padEnd(labelWidth))
break;
break
case CheckStatus.Timeout:
statusLabel = chalk.yellow("TIMEOUT".padEnd(labelWidth))
break;
break
case CheckStatus.NonSuccessCode:
case CheckStatus.GenericError:
statusLabel = chalk.red("BROKEN".padEnd(labelWidth))
break;
break
}
if (check.status != CheckStatus.Skipped) {
......@@ -149,7 +173,7 @@ export class ConsoleReporter implements IReporter {
}
}
process(pages: Map<string, ResultItem[]>) {
process(pages: Map<string, ResultItem[]>): void {
let allSkipped = 0
let allOks = 0
......
......
......@@ -8,7 +8,7 @@ export class Result {
constructor(readonly ignoreSkipped: boolean, readonly disablePrint: boolean) { }
public add(completedCheck: ResultItem, parent: string = "original request") {
public add(completedCheck: ResultItem, parent = "original request"): void {
if (completedCheck.status == CheckStatus.Skipped && this.ignoreSkipped) {
return
}
......@@ -22,6 +22,7 @@ export class Result {
}
if (this.pages.has(parent)) {
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
this.pages.get(parent)!.push(completedCheck)
} else {
this.pages.set(parent, [completedCheck])
......@@ -49,15 +50,15 @@ export class Result {
return count
}
public report<ReporterT extends IReporter>(reporter: ReporterT): any {
public report<ReporterT extends IReporter>(reporter: ReporterT): unknown {
return reporter.process(this.pages)
}
public success() {
public success(): boolean {
return !this.atLeastOneBroken
}
public set(pages: Map<string, ResultItem[]>) {
public set(pages: Map<string, ResultItem[]>): void {
this.pages = pages
}
}
......
......
import { Inspector, URLsMatchingSet, URLMatchingRule, Config } from "../src/inspector"
import { expect, assert } from "chai";
import "mocha";
import { expect, assert } from "chai"
import "mocha"
describe("extractURLs", () => {
......@@ -9,27 +9,27 @@ describe("extractURLs", () => {
it("works for <a href=...>", () => {
const result = new Inspector(new URLsMatchingSet(URLMatchingRule.AHRef), new Config()).extractURLs(`<html><a href="${url}">Text</a></html>`)
expect(result).to.eql(new Set([url]))
});
})
it("works for <script src=...>", () => {
const result = new Inspector(new URLsMatchingSet(URLMatchingRule.ScriptSrc), new Config()).extractURLs(`<html><script src="${url}">Text</script></html>`)
expect(result).to.eql(new Set([url]))
});
})
it("works for <link href=...>", () => {
const result = new Inspector(new URLsMatchingSet(URLMatchingRule.LinkHref), new Config()).extractURLs(`<html><link href="${url}"></link></html>`)
expect(result).to.eql(new Set([url]))
});
})
it("works for <img src=...>", () => {
const result = new Inspector(new URLsMatchingSet(URLMatchingRule.ImgSrc), new Config()).extractURLs(`<html><img src="${url}">Text</img></html>`)
expect(result).to.eql(new Set([url]))
});
})
it("works for <iframe src=...>", () => {
const result = new Inspector(new URLsMatchingSet(URLMatchingRule.IFrameSrc), new Config()).extractURLs(`<html><iframe src="${url}">Text</iframe></html>`)
expect(result).to.eql(new Set([url]))
});
})
it("works for many rules", () => {
const result = new Inspector(new URLsMatchingSet(), new Config())
......@@ -42,7 +42,7 @@ describe("extractURLs", () => {
</html>`
)
expect(result).to.eql(new Set(["1", "2", "3", "4"]))
});
})
it("does not match unless rule supplied", () => {
const result = new Inspector(new URLsMatchingSet(URLMatchingRule.ImgSrc), new Config())
......@@ -53,7 +53,7 @@ describe("extractURLs", () => {
</html>`
)
expect(result).to.eql(new Set([url]))
});
})
it("filters duplicates", () => {
const result = new Inspector(new URLsMatchingSet(), new Config())
......@@ -65,10 +65,10 @@ describe("extractURLs", () => {
</html>`
)
expect(result).to.eql(new Set([url, "another-url"]))
});
})
it("fails for unknown rule", () => {
assert.throws(() => new Inspector(new URLsMatchingSet("error" as URLMatchingRule), new Config()).extractURLs(`<html><img src="${url}">Text</img></html>`), /unknown/)
});
})
});
})
import { Inspector, URLsMatchingSet, Config, IHttpClient, HttpClientFailure, AxiosHttpClient } from "../src/inspector"
import { assert } from "chai";
import "mocha";
import { ConsoleReporter, JUnitReporter, IReporter } from "../src/report";
import { ResultItem, CheckStatus, Result } from "../src/result";
import intercept from "intercept-stdout";
import { assert } from "chai"
import "mocha"
import { ConsoleReporter, JUnitReporter, IReporter } from "../src/report"
import { ResultItem, CheckStatus, Result } from "../src/result"
import intercept from "intercept-stdout"
class MockHttpClient implements IHttpClient {
......@@ -11,6 +11,7 @@ class MockHttpClient implements IHttpClient {
constructor(readonly map: Map<string, [string[], boolean, boolean, number]>) { }
async request(get: boolean, url: string): Promise<string> {
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
const [urls, timeout, failure, code] = this.map.get(url)!
if (timeout) {
throw new HttpClientFailure(true, -1)
......@@ -27,16 +28,17 @@ class MockHttpClient implements IHttpClient {
}
class MockReporter implements IReporter {
process(pages: Map<string, ResultItem[]>): any {
process(pages: Map<string, ResultItem[]>): Map<string, ResultItem[]> {
return pages
}
}
function toURL(url: string, path: string = "") {
function toURL(url: string, path = "") {
return new URL(`https://${url}/${path}`).href
}
function stripEffects(text: string) {
// eslint-disable-next-line no-control-regex
return text.replace(/[\u001b\u009b][[()#;?]*(?:[0-9]{1,4}(?:;[0-9]{0,4})*)?[0-9A-ORZcf-nqry=><]/g, "")
}
......@@ -44,12 +46,15 @@ function assertEqualResults(expected: Map<string, ResultItem[]>, actual: Map<str
for (const [expectedURL, expectedChecks] of expected) {
assert(actual.has(expectedURL))
let actualChecks = actual.get(expectedURL)!
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
const actualChecks = actual.get(expectedURL)!
assert(expectedChecks.length == actualChecks.length)
for (const expectedCheck of expectedChecks) {
let actualCheck = actualChecks.find(c => c.url === expectedCheck.url)
const actualCheck = actualChecks.find(c => c.url === expectedCheck.url)
assert(actualCheck)
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
assert(expectedCheck.status == actualCheck!.status)
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
assert(expectedCheck.message == actualCheck!.message)
}
}
......@@ -102,7 +107,7 @@ describe("Axios web server", async () => {
try {
await new AxiosHttpClient(5, []).request(false, "https://dbogatov.org")
} catch (exception) {
const error: HttpClientFailure = exception;
const error: HttpClientFailure = exception
assert(error.timeout)
}
})
......@@ -111,7 +116,7 @@ describe("Axios web server", async () => {
try {
await new AxiosHttpClient(2000, []).request(false, "https://dbogatov.org/not-found-123")
} catch (exception) {
const error: HttpClientFailure = exception;
const error: HttpClientFailure = exception
assert(error.code == 404)
}
})
......@@ -120,7 +125,7 @@ describe("Axios web server", async () => {
try {
await new AxiosHttpClient(1000, []).request(false, "bad-url")
} catch (exception) {
const error: HttpClientFailure = exception;
const error: HttpClientFailure = exception
assert(!error.timeout)
assert(error.code == -1)
}
......@@ -135,7 +140,7 @@ describe("process mock URL", function () {
([true, false] as boolean[]).forEach(recursive => {
it(`processes ${recursive ? "" : "non-"}recursive`, async () => {
let config = new Config()
const config = new Config()
config.disablePrint = true
config.skipURLs = ["to-skip"]
const inspector = new Inspector(
......@@ -143,14 +148,14 @@ describe("process mock URL", function () {
config,
httpClient
)
let unhook_intercept = intercept(_ => { return "" });
const unhook_intercept = intercept(_ => { return "" })
const result = await inspector.processURL(new URL("https://original.com"), recursive)
unhook_intercept();
unhook_intercept()
const actual = result.report(new MockReporter()) as Map<string, ResultItem[]>
let expected = new Map(expectedNonRecursive)
const expected = new Map(expectedNonRecursive)
if (recursive) {
expected.set(
......@@ -164,26 +169,26 @@ describe("process mock URL", function () {
assertEqualResults(expected, actual)
assert(!result.success())
});
})
})
describe("reporters", function () {
it("console", () => {
let log: string = ""
let unhook_intercept = intercept(line => {
let log = ""
const unhook_intercept = intercept(line => {
log += stripEffects(line)
return ""
});
})
let result = new Result(true, true)
const result = new Result(true, true)
result.set(expectedNonRecursive)
result.report(new ConsoleReporter())
unhook_intercept();
unhook_intercept()
let lines = log.split(/\r?\n/)
const lines = log.split(/\r?\n/)
for (const [expectedURL, expectedChecks] of expectedNonRecursive) {
assert(lines.find(l => l.startsWith(expectedURL)))
......@@ -192,13 +197,15 @@ describe("process mock URL", function () {
if (expectedCheck.status == CheckStatus.Skipped) {
continue
}
let check = lines.find(l => l.includes("\t") && l.includes(expectedCheck.url + " "))
const check = lines.find(l => l.includes("\t") && l.includes(expectedCheck.url + " "))
assert(check, `${expectedCheck.url} not found`)
assert(
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
check!.includes(expectedCheck.status == CheckStatus.NonSuccessCode || expectedCheck.status == CheckStatus.GenericError ? "BROKEN" : expectedCheck.status),
`${expectedCheck.url}: status (${expectedCheck.status}) not found in "${check}"`
)
if (expectedCheck.message) {
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
assert(check!.includes(expectedCheck.message))
}
}
......@@ -207,21 +214,21 @@ describe("process mock URL", function () {
it("junit", () => {
let log: string = ""
let unhook_intercept = intercept(line => {
let log = ""
const unhook_intercept = intercept(line => {
log += line
return ""
});
})
let result = new Result(true, true)
const result = new Result(true, true)
result.set(expectedNonRecursive)
result.report(new JUnitReporter(false))
unhook_intercept();
unhook_intercept()
result.report(new JUnitReporter())
let lines = log.split(/\r?\n/)
const lines = log.split(/\r?\n/)
for (const [expectedURL, expectedChecks] of expectedNonRecursive) {
assert(lines.find(l => l.includes("testsuite") && l.includes(expectedURL)))
......@@ -235,7 +242,7 @@ describe("process mock URL", function () {
})
describe("process real URL", async () => {
let config = new Config()
const config = new Config()
config.disablePrint = true
const inspector = new Inspector(
new URLsMatchingSet(),
......@@ -248,29 +255,29 @@ describe("process real URL", async () => {
describe("result", () => {
it("ignores skipped", () => {
let result = new Result(true, true)
const result = new Result(true, true)
result.add({ status: CheckStatus.Skipped, url: "skip" })
result.add(new ResultItem())
assert(result.count() == 1)
})
it("print progress", () => {
let result = new Result(true, false)
const result = new Result(true, false)
let log: string = ""
let unhook_intercept = intercept(line => {
let log = ""
const unhook_intercept = intercept(line => {
log += line
return ""
});
})
result.add({ status: CheckStatus.GenericError, url: "" })
for (let index = 0; index < 120; index++) {
result.add({ status: CheckStatus.OK, url: `${index}` })
}
unhook_intercept();
unhook_intercept()
let lines = log.split(/\r?\n/)
const lines = log.split(/\r?\n/)
assert(result.count() == 121)
assert(lines.length == 2)
......
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please to comment