1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
|
// Package main provides a tool to generate Unbound DNS server configuration
// for domain blocking. It fetches domain lists from URLs and converts them
// into Unbound local-zone configuration format.
//
// Usage:
//
// unbound-ads <url-list> <output-file>
//
// The url-list should contain HTTP(S) URLs, one per line, pointing to files
// containing domain lists. Comments (lines starting with #) are ignored.
// The output file will contain Unbound configuration in the format:
//
// local-zone: "domain.com" refuse
package main
import (
"bufio"
"flag"
"fmt"
"log/slog"
"net/http"
"os"
"strings"
)
// Version is set during build using ldflags
var Version = "dev-" + strings.Split(strings.TrimPrefix(`$Format:%H$`, "$Format:"), "-")[0]
// main processes a list of URLs containing domain lists and generates
// an Unbound DNS server configuration file for domain blocking.
func main() {
showVersion := flag.Bool("version", false, "Show version information")
flag.Parse()
if *showVersion {
fmt.Println("unbound-ads version:", Version)
return
}
args := flag.Args()
if len(args) != 2 {
slog.Error("usage: unbound-ads <url-list> <output-file>")
os.Exit(1)
}
urls, err := fetchURLList(args[0])
if err != nil {
slog.Error("failed to fetch URL list", "error", err)
os.Exit(1)
}
f, err := os.Create(args[1])
if err != nil {
slog.Error("failed to create output file", "error", err)
os.Exit(1)
}
defer f.Close()
w := bufio.NewWriter(f)
defer w.Flush()
domains := make(map[string]struct{})
for i, url := range urls {
slog.Info("fetching domains", "url", url, "progress", fmt.Sprintf("%d/%d", i+1, len(urls)))
if err := fetchDomainsAndWrite(url, w, domains); err != nil {
slog.Warn("failed to process url", "url", url, "error", err)
continue
}
}
slog.Info("completed", "total_domains", len(domains))
}
// fetchURLList retrieves a list of URLs from the specified URL.
// It ignores empty lines and comments (lines starting with #).
// Returns a slice of URLs or an error if the fetch fails.
func fetchURLList(url string) ([]string, error) {
resp, err := http.Get(url)
if err != nil {
return nil, fmt.Errorf("http get failed: %w", err)
}
defer resp.Body.Close()
var urls []string
scanner := bufio.NewScanner(resp.Body)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" || strings.HasPrefix(line, "#") {
continue
}
urls = append(urls, line)
}
return urls, scanner.Err()
}
// fetchDomainsAndWrite retrieves domains from the specified URL and writes them
// to the output in Unbound configuration format. It handles various domain list
// formats including "0.0.0.0 domain.com" and plain domain lists.
// Domains are deduplicated using the seen map.
// Returns an error if fetching or writing fails.
func fetchDomainsAndWrite(url string, w *bufio.Writer, seen map[string]struct{}) error {
resp, err := http.Get(url)
if err != nil {
return fmt.Errorf("http get failed: %w", err)
}
defer resp.Body.Close()
fmt.Fprint(w, "server:\n")
var count int
scanner := bufio.NewScanner(resp.Body)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
// Skip empty lines and comments
if line == "" || strings.HasPrefix(line, "#") {
continue
}
// Handle both "0.0.0.0 domain.com" and plain domain formats
var domain string
if strings.Contains(line, " ") {
parts := strings.Fields(line)
if len(parts) >= 2 {
domain = parts[1]
}
} else {
domain = line
}
// Normalize and validate domain format
domain = strings.ToLower(strings.TrimSpace(domain))
if domain == "" || !strings.Contains(domain, ".") || strings.HasPrefix(domain, ".") || strings.HasSuffix(domain, ".") {
continue
}
// Deduplicate domains
if _, exists := seen[domain]; exists {
continue
}
seen[domain] = struct{}{}
count++
if _, err := fmt.Fprintf(w, " local-zone: %q refuse\n", domain); err != nil {
return fmt.Errorf("failed to write domain: %w", err)
}
}
if err := scanner.Err(); err != nil {
return fmt.Errorf("scanner error: %w", err)
}
slog.Info("processed url", "url", url, "new_domains", count)
return nil
}
|
