diff options
| author | Max Resnick <max@ofmax.li> | 2025-03-26 22:21:03 -0700 |
|---|---|---|
| committer | Max Resnick <max@ofmax.li> | 2025-03-26 22:21:03 -0700 |
| commit | 3a9d7a36a85cf8ae1098f213186ef83bbc3c56ba (patch) | |
| tree | 96866fc8b7fdd80f4f04bad2631f267ba5671d9c | |
| parent | 45220cc2793915008ae7b41f6f533ae86f1c56c3 (diff) | |
| download | unbound-adblock-config-master.tar.gz | |
Diffstat (limited to '')
| -rw-r--r-- | main.go | 16 | ||||
| -rw-r--r-- | main_test.go | 31 |
2 files changed, 33 insertions, 14 deletions
@@ -59,6 +59,11 @@ func main() { w := bufio.NewWriter(f) defer w.Flush() + if _, err := fmt.Fprintln(w, "server:"); err != nil { + slog.Error("failed to write header", "error", err) + os.Exit(1) + } + domains := make(map[string]struct{}) for i, url := range urls { slog.Info("fetching domains", "url", url, "progress", fmt.Sprintf("%d/%d", i+1, len(urls))) @@ -104,7 +109,6 @@ func fetchDomainsAndWrite(url string, w *bufio.Writer, seen map[string]struct{}) } defer resp.Body.Close() - fmt.Fprint(w, "server:\n") var count int scanner := bufio.NewScanner(resp.Body) for scanner.Scan() { @@ -127,6 +131,10 @@ func fetchDomainsAndWrite(url string, w *bufio.Writer, seen map[string]struct{}) // Normalize and validate domain format domain = strings.ToLower(strings.TrimSpace(domain)) + if idx := strings.Index(domain, "$"); idx != -1 { + domain = domain[:idx] + } + domain = strings.Trim(domain, "|^") if domain == "" || !strings.Contains(domain, ".") || strings.HasPrefix(domain, ".") || strings.HasSuffix(domain, ".") { continue } @@ -138,7 +146,7 @@ func fetchDomainsAndWrite(url string, w *bufio.Writer, seen map[string]struct{}) seen[domain] = struct{}{} count++ - if _, err := fmt.Fprintf(w, " local-zone: %q refuse\n", domain); err != nil { + if _, err := fmt.Fprintf(w, "\tlocal-zone: %q refuse\n", domain); err != nil { return fmt.Errorf("failed to write domain: %w", err) } } @@ -147,6 +155,10 @@ func fetchDomainsAndWrite(url string, w *bufio.Writer, seen map[string]struct{}) return fmt.Errorf("scanner error: %w", err) } + if err := w.Flush(); err != nil { + return fmt.Errorf("failed to flush writer: %w", err) + } + slog.Info("processed url", "url", url, "new_domains", count) return nil } diff --git a/main_test.go b/main_test.go index 64c87cd..cd86d0b 100644 --- a/main_test.go +++ b/main_test.go @@ -75,10 +75,7 @@ func TestFetchDomainsAndWrite(t *testing.T) { content: `# comment 0.0.0.0 domain1.com 0.0.0.0 domain2.com`, - want: `server: - local-zone: "domain1.com" refuse - local-zone: "domain2.com" refuse -`, + want: "\tlocal-zone: \"domain1.com\" refuse\n\tlocal-zone: \"domain2.com\" refuse\n", wantSeen: map[string]struct{}{ "domain1.com": {}, "domain2.com": {}, @@ -89,10 +86,7 @@ func TestFetchDomainsAndWrite(t *testing.T) { content: `# comment domain1.com domain2.com`, - want: `server: - local-zone: "domain1.com" refuse - local-zone: "domain2.com" refuse -`, + want: "\tlocal-zone: \"domain1.com\" refuse\n\tlocal-zone: \"domain2.com\" refuse\n", wantSeen: map[string]struct{}{ "domain1.com": {}, "domain2.com": {}, @@ -104,15 +98,28 @@ domain2.com`, 0.0.0.0 domain1.com 0.0.0.0 DOMAIN1.COM domain2.com`, - want: `server: - local-zone: "domain1.com" refuse - local-zone: "domain2.com" refuse -`, + want: "\tlocal-zone: \"domain1.com\" refuse\n\tlocal-zone: \"domain2.com\" refuse\n", wantSeen: map[string]struct{}{ "domain1.com": {}, "domain2.com": {}, }, }, + { + name: "domains with pipe and caret", + content: `||domain1.com^ +||domain2.com^ +0.0.0.0 ||domain3.com^ +||sub.domain4.com^$important +||domain5.com^$third-party`, + want: "\tlocal-zone: \"domain1.com\" refuse\n\tlocal-zone: \"domain2.com\" refuse\n\tlocal-zone: \"domain3.com\" refuse\n\tlocal-zone: \"sub.domain4.com\" refuse\n\tlocal-zone: \"domain5.com\" refuse\n", + wantSeen: map[string]struct{}{ + "domain1.com": {}, + "domain2.com": {}, + "domain3.com": {}, + "sub.domain4.com": {}, + "domain5.com": {}, + }, + }, } for _, tt := range tests { |