1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
|
// Package safefetch makes outbound requests to addresses this server did not choose.
//
// Everything listend fetches is named by somebody else. The feed URL comes from the phone, and
// the enclosure and <podcast:transcript> URLs inside the feed come from the publisher. listend
// listens on 127.0.0.1:38090 behind Apache on a host running other things, so a feed naming a
// loopback, private or link-local address would have the server fetch something reachable only
// from inside -- and in the transcript case store the result and hand it to the phone as an
// episode's words.
//
// The check happens when the connection is dialled rather than when the URL is read. That is
// the only point that also covers a redirect into an internal address and a hostname that
// resolves to one: the address connected to is the address that was checked.
package safefetch
import (
"context"
"fmt"
"net"
"net/http"
"net/netip"
"net/url"
"time"
)
// DefaultTimeout is what Client uses when given none.
const DefaultTimeout = 60 * time.Second
// CheckURL rejects anything that is not an ordinary web address.
//
// A publisher naming file:// or gopher:// has not given us a transcript, and Go's http client
// would not fetch those anyway -- the value of saying so here is that the refusal is explicit
// and testable rather than an obscure transport error.
func CheckURL(raw string) error {
parsed, err := url.Parse(raw)
if err != nil {
return fmt.Errorf("safefetch: %q is not a URL: %w", raw, err)
}
if parsed.Scheme != "http" && parsed.Scheme != "https" {
return fmt.Errorf("safefetch: %q is not http or https", raw)
}
if parsed.Host == "" {
return fmt.Errorf("safefetch: %q names no host", raw)
}
return nil
}
// Client returns an http.Client that can only reach public addresses.
func Client(timeout time.Duration) *http.Client {
if timeout <= 0 {
timeout = DefaultTimeout
}
dialer := &net.Dialer{Timeout: 20 * time.Second, KeepAlive: 30 * time.Second}
transport := http.DefaultTransport.(*http.Transport).Clone()
// No proxy. http.DefaultTransport carries ProxyFromEnvironment, and with HTTP_PROXY set
// the connection goes to the proxy instead -- so the address checked below would be the
// proxy's and the one that matters would never be looked at at all.
transport.Proxy = nil
transport.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
host, port, err := net.SplitHostPort(addr)
if err != nil {
return nil, fmt.Errorf("safefetch: %q: %w", addr, err)
}
addresses, err := net.DefaultResolver.LookupIPAddr(ctx, host)
if err != nil {
return nil, fmt.Errorf("safefetch: %q: %w", host, err)
}
// Dial a checked address rather than the hostname, so the connection cannot land
// somewhere a second lookup would have resolved to.
var lastErr error
for _, candidate := range addresses {
if !permitted(candidate.IP) {
lastErr = fmt.Errorf("safefetch: %s is not a public address", candidate.IP)
continue
}
conn, err := dialer.DialContext(ctx, network, net.JoinHostPort(candidate.IP.String(), port))
if err == nil {
return conn, nil
}
lastErr = err
}
if lastErr == nil {
lastErr = fmt.Errorf("safefetch: %q resolved to nothing", host)
}
return nil, lastErr
}
return &http.Client{Timeout: timeout, Transport: transport}
}
// specialPurpose are the ranges that are not the open internet, from the IANA special-purpose
// registries.
//
// An allowlist rather than a list of things that felt private: "not RFC 1918" is not the same
// as "public". 100.64.0.0/10 is carrier-grade NAT and routes to a real host on a real network;
// 64:ff9b::/96 is NAT64 and carries an IPv4 address inside it, 127.0.0.1 included; 2002::/16 is
// 6to4 and does the same. None of them is private by Go's definition and none of them is
// somewhere a podcast is served from.
var specialPurpose = []netip.Prefix{
netip.MustParsePrefix("0.0.0.0/8"), // "this network"
netip.MustParsePrefix("100.64.0.0/10"), // carrier-grade NAT
netip.MustParsePrefix("192.0.0.0/24"), // IETF protocol assignments
netip.MustParsePrefix("192.0.2.0/24"), // documentation
netip.MustParsePrefix("192.88.99.0/24"), // 6to4 relay anycast, deprecated
netip.MustParsePrefix("198.18.0.0/15"), // benchmarking
netip.MustParsePrefix("198.51.100.0/24"), // documentation
netip.MustParsePrefix("203.0.113.0/24"), // documentation
netip.MustParsePrefix("240.0.0.0/4"), // reserved, and 255.255.255.255 within it
netip.MustParsePrefix("::/96"), // IPv4-compatible IPv6, deprecated
netip.MustParsePrefix("64:ff9b::/96"), // NAT64
netip.MustParsePrefix("64:ff9b:1::/48"), // local-use NAT64
netip.MustParsePrefix("100::/64"), // discard-only
netip.MustParsePrefix("2001::/23"), // IETF protocol assignments, Teredo within it
netip.MustParsePrefix("2001:db8::/32"), // documentation
netip.MustParsePrefix("2002::/16"), // 6to4, which embeds an IPv4 address
netip.MustParsePrefix("3fff::/20"), // documentation
}
// permitted reports whether an address is one the open internet could have given us.
//
// Stated as what is allowed rather than as a list of things to block: an ordinary global
// unicast address that is not in a special-purpose range. Everything a podcast is actually
// served from is one of those, and the failure mode of being too strict is a feed that does not
// mirror, which is visible -- while the failure mode of being too lax is the server fetching
// something only it can reach and handing the result to the phone.
func permitted(ip net.IP) bool {
if ip == nil {
return false
}
addr, ok := netip.AddrFromSlice(ip)
if !ok {
return false
}
// An IPv4 address written in IPv6-mapped form is the same address and reaches the same
// host, so it is judged as the IPv4 one.
addr = addr.Unmap()
if !addr.IsGlobalUnicast() {
// Loopback, unspecified, multicast and link-local are all excluded here.
return false
}
if addr.IsPrivate() {
return false
}
for _, prefix := range specialPurpose {
if prefix.Contains(addr) {
return false
}
}
return true
}
|
