(cache)extensions/LeanSERP-Studio-package/LeanSERP-Studio-URL-Mode/Find-PslHostCandidates.ps1 at main · Fengsemul/extensions · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
[CmdletBinding()]
param(
    [Parameter(Mandatory = $true)]
    [string]$InputFile,

    [Parameter(Mandatory = $false)]
    [string]$PublicSuffixList =
        "$PSScriptRoot\public_suffix_list.dat",

    [Parameter(Mandatory = $false)]
    [string]$OutputDirectory =
        "$env:USERPROFILE\Downloads\LeanSERP-PSL-Candidates"
)

Set-StrictMode -Version Latest
$ErrorActionPreference = "Stop"

function New-Utf8Writer {
    param(
        [Parameter(Mandatory = $true)]
        [string]$Path
    )

    return [System.IO.StreamWriter]::new(
        $Path,
        $false,
        [System.Text.UTF8Encoding]::new($false),
        1048576
    )
}

function Remove-BomArtifacts {
    param(
        [AllowEmptyString()]
        [string]$Value
    )

    $text = [string]$Value
    $text = $text.TrimStart([char]0xFEFF)

    $mojibakeBom = [string]::Concat(
        [char]0x00EF,
        [char]0x00BB,
        [char]0x00BF
    )

    $replacementBom = [string]::Concat(
        [char]0x00EF,
        [char]0x00BF,
        [char]0x00BD
    )

    while ($text.StartsWith($mojibakeBom)) {
        $text = $text.Substring(
            $mojibakeBom.Length
        )
    }

    while ($text.StartsWith($replacementBom)) {
        $text = $text.Substring(
            $replacementBom.Length
        )
    }

    return $text
}

function ConvertTo-NormalizedHostname {
    param(
        [AllowEmptyString()]
        [string]$Line
    )

    if ($null -eq $Line) {
        return ""
    }

    $value = Remove-BomArtifacts -Value $Line
    $value = $value.Trim()

    if (
        [string]::IsNullOrWhiteSpace($value) -or
        $value.StartsWith("#") -or
        $value.StartsWith("!") -or
        $value.Contains([char]0)
    ) {
        return ""
    }

    $fields = $value -split "\s+"

    if (
        $fields.Count -ge 2 -and
        $fields[0] -match
            "^(?:0\.0\.0\.0|127\.0\.0\.1|::1)$"
    ) {
        $value = $fields[1]
    }
    else {
        $value = $fields[0]
    }

    if (
        [string]::IsNullOrWhiteSpace($value) -or
        $value.StartsWith("@@")
    ) {
        return ""
    }

    $value = $value.ToLowerInvariant()

    if ($value.StartsWith("*://*.")) {
        $value = $value.Substring(6)
    }
    elseif ($value.StartsWith("*://")) {
        $value = $value.Substring(4)

        if ($value.StartsWith("*.")) {
            $value = $value.Substring(2)
        }
    }
    elseif (
        $value -match
        "^[a-z][a-z0-9+.-]*://"
    ) {
        try {
            $uri = [System.Uri]$value
            $value = $uri.DnsSafeHost
        }
        catch {
            return ""
        }
    }

    $value = Remove-BomArtifacts -Value $value
    $value = $value -replace "[/?#].*$", ""
    $value = $value.Trim(".")
    $value = $value.ToLowerInvariant()

    if ($value.StartsWith("www.")) {
        $value = $value.Substring(4)
    }

    if (
        [string]::IsNullOrWhiteSpace($value) -or
    inputFile = (
        Resolve-Path `
            -LiteralPath $InputFile
    ).Path
    inputByteLength = (
        Get-Item `
            -LiteralPath $InputFile
    ).Length
    inputLineCount = $totalLines
    validHostnameCount =
        $validHostnames
    rejectedLineCount =
        $rejectedLines
    publicSuffixRuleCount =
        $publicSuffixes.Count
    candidateOccurrences =
        $candidateOccurrences
    uniqueCandidateCount =
        $candidates.Count
    candidateSha256 =
        $candidateHash
    networkTestsPerformed =
        $false
}

$metadataJson =
    $metadata |
    ConvertTo-Json -Depth 8

[System.IO.File]::WriteAllText(
    $metadataPath,
    $metadataJson +
        [Environment]::NewLine,
    [System.Text.UTF8Encoding]::new(
        $false
    )
)

Write-Host ""
Write-Host (
    "PSL-host candidate scan completed."
) -ForegroundColor Green

Write-Host (
    "Input lines: {0:N0}" -f
    $totalLines
)

Write-Host (
    "Valid hostnames: {0:N0}" -f
    $validHostnames
)

Write-Host (
    "Rejected lines: {0:N0}" -f
    $rejectedLines
)

Write-Host (
    "Unique PSL-host candidates: {0:N0}" -f
    $candidates.Count
) -ForegroundColor Green

Write-Host ""
Write-Host "Candidates:"
Write-Host $candidatePath
Write-Host "Candidate details:"
Write-Host $detailPath
Write-Host "Rejected-input report:"
Write-Host $rejectedPath
Write-Host "Metadata:"
Write-Host $metadataPath
Write-Host "Candidate SHA-256:"
Write-Host $candidateHash