(cache)extensions/LeanSERP-Studio-package/LeanSERP-Studio-URL-Mode/LeanSERP-Studio.ps1 at main · Fengsemul/extensions · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
[CmdletBinding()]
param(
    [Parameter(Mandatory = $false)]
    [string[]]$InputFile,

    [Parameter(Mandatory = $false)]
    [string]$OutputDirectory =
        "$env:USERPROFILE\Downloads\LeanSERP-Output",

    [Parameter(Mandatory = $false)]
    [ValidateRange(1000, 1000000)]
    [int]$ChunkSize = 250000,

    [Parameter(Mandatory = $false)]
    [switch]$KeepUnderscores,

    [Parameter(Mandatory = $false)]
    [switch]$KeepTemporaryFiles,

    [Parameter(Mandatory = $false)]
    [string]$ProgressLogPath = ""

)

Set-StrictMode -Version Latest
$ErrorActionPreference = "Stop"
$script:ProgressWriter = $null

if (
    -not [string]::IsNullOrWhiteSpace(
        $ProgressLogPath
    )
) {
    $progressDirectory =
        Split-Path -Parent $ProgressLogPath

    if (
        -not [string]::IsNullOrWhiteSpace(
            $progressDirectory
        )
    ) {
        New-Item `
            -ItemType Directory `
            -Path $progressDirectory `
            -Force |
            Out-Null
    }

    $script:ProgressWriter =
        [System.IO.StreamWriter]::new(
            $ProgressLogPath,
            $true,
            [System.Text.UTF8Encoding]::new(
                $false
            ),
            4096
        )

    $script:ProgressWriter.AutoFlush =
        $true
}

function Write-StudioProgress {
    param(
        [AllowEmptyString()]
        [string]$Text = ""
    )

    Write-Host $Text

    if (
        $null -ne
        $script:ProgressWriter
    ) {
        $script:ProgressWriter.WriteLine(
            $Text
        )
    }
}

$FormatName = "leanserp-label-package"
$FormatVersion = 1
$MaximumLabelLength = 63
$RejectionSampleLimit = 100

function New-Utf8NoBomWriter {
    param(
        [Parameter(Mandatory = $true)]
        [string]$Path
    )

    $encoding = [System.Text.UTF8Encoding]::new($false)

    return [System.IO.StreamWriter]::new(
        $Path,
        $false,
        $encoding,
        1048576
    )
}

function Get-Sha256 {
    param(
        [Parameter(Mandatory = $true)]
        [string]$Path
    )

    return (
        Get-FileHash `
            -LiteralPath $Path `
            -Algorithm SHA256
    ).Hash.ToLowerInvariant()
}

function ConvertTo-LeanSerpLabel {
    param(
        [AllowEmptyString()]
        [string]$Line,

        [Parameter(Mandatory = $true)]
        [bool]$AllowUnderscore
    )

    $original = [string]$Line

    if ($null -eq $Line) {
        return [pscustomobject]@{
            Accepted = $false
            Label = ""
            Reason = "empty-line"
            Original = ""
        }
    }

        $value = $Line.TrimStart([char]0xFEFF).Trim()

    if ($value.StartsWith([string]([char]0x00EF) + [char]0x00BB + [char]0x00BF)) {
        $value = $value.Substring(3)
    }

    if ([string]::IsNullOrWhiteSpace($value)) {
        return [pscustomobject]@{
            Accepted = $false
            Label = ""
            Reason = "empty-line"

[System.IO.File]::WriteAllText(
    $outputMetadataPath,
    $metadataJson + [Environment]::NewLine,
    [System.Text.UTF8Encoding]::new($false)
)

if (-not $KeepTemporaryFiles) {
    Remove-Item `
        -LiteralPath $temporaryDirectory `
        -Recurse `
        -Force
}

Write-StudioProgress

Write-StudioProgress `
    -Text "LeanSERP Studio build completed."

Write-StudioProgress `
    -Text (
        "Unique labels: {0:N0}" -f
        $mergeResult.OutputCount
    )

Write-StudioProgress `
    -Text (
        "Rejected lines: {0:N0}" -f
        $totalRejectedLines
    )

Write-StudioProgress `
    -Text (
        "Duplicate labels removed: {0:N0}" -f
        (
            $duplicatesRemovedInChunks +
            $mergeResult.DuplicateCount
        )
    )

Write-StudioProgress `
    -Text "Label package:"

Write-StudioProgress `
    -Text $outputLabelPath

Write-StudioProgress `
    -Text "Metadata:"

Write-StudioProgress `
    -Text $outputMetadataPath

Write-StudioProgress `
    -Text "Rejection report:"

Write-StudioProgress `
    -Text $outputRejectionPath

Write-StudioProgress `
    -Text "SHA-256:"

Write-StudioProgress `
    -Text $outputHash

if ($null -ne $script:ProgressWriter) {
    try {
        $script:ProgressWriter.Flush()
        $script:ProgressWriter.Dispose()
    }
    catch {
    }

    $script:ProgressWriter = $null
}