Add HTTP proxy option for subscriptions

pull/790/head
Kebin Liu 4 years ago committed by GitHub
parent 0f258fd55b
commit cf7712acea
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 1
      api/feed.go
  2. 26
      api/payload.go
  3. 1
      api/subscription.go
  4. 14
      config/options.go
  5. 2
      config/parser.go
  6. 2
      database/migration.go
  7. 3
      database/sql.go
  8. 1
      database/sql/schema_version_35.sql
  9. 22
      http/client/client.go
  10. 33
      locale/translations.go
  11. 1
      locale/translations/de_DE.json
  12. 1
      locale/translations/en_US.json
  13. 1
      locale/translations/es_ES.json
  14. 1
      locale/translations/fr_FR.json
  15. 1
      locale/translations/it_IT.json
  16. 1
      locale/translations/ja_JP.json
  17. 1
      locale/translations/nl_NL.json
  18. 1
      locale/translations/pl_PL.json
  19. 1
      locale/translations/pt_BR.json
  20. 1
      locale/translations/ru_RU.json
  21. 1
      locale/translations/zh_CN.json
  22. 5
      miniflux.1
  23. 4
      model/feed.go
  24. 2
      model/feed_test.go
  25. 21
      reader/feed/handler.go
  26. 12
      reader/icon/finder.go
  27. 7
      reader/subscription/finder.go
  28. 17
      storage/feed.go
  29. 2
      template/common.go
  30. 3
      template/html/add_subscription.html
  31. 3
      template/html/choose_subscription.html
  32. 3
      template/html/edit_feed.html
  33. 15
      template/views.go
  34. 3
      ui/feed_edit.go
  35. 3
      ui/form/feed.go
  36. 34
      ui/form/subscription.go
  37. 6
      ui/subscription_add.go
  38. 4
      ui/subscription_bookmarklet.go
  39. 1
      ui/subscription_choose.go
  40. 5
      ui/subscription_submit.go

@ -51,6 +51,7 @@ func (h *handler) createFeed(w http.ResponseWriter, r *http.Request) {
feedInfo.Password,
feedInfo.ScraperRules,
feedInfo.RewriteRules,
feedInfo.FetchViaProxy,
)
if err != nil {
json.ServerError(w, r, err)

@ -24,21 +24,23 @@ type entriesResponse struct {
}
type feedCreation struct {
FeedURL string `json:"feed_url"`
CategoryID int64 `json:"category_id"`
UserAgent string `json:"user_agent"`
Username string `json:"username"`
Password string `json:"password"`
Crawler bool `json:"crawler"`
ScraperRules string `json:"scraper_rules"`
RewriteRules string `json:"rewrite_rules"`
FeedURL string `json:"feed_url"`
CategoryID int64 `json:"category_id"`
UserAgent string `json:"user_agent"`
Username string `json:"username"`
Password string `json:"password"`
Crawler bool `json:"crawler"`
FetchViaProxy bool `json:"fetch_via_proxy"`
ScraperRules string `json:"scraper_rules"`
RewriteRules string `json:"rewrite_rules"`
}
type subscriptionDiscovery struct {
URL string `json:"url"`
UserAgent string `json:"user_agent"`
Username string `json:"username"`
Password string `json:"password"`
URL string `json:"url"`
UserAgent string `json:"user_agent"`
Username string `json:"username"`
Password string `json:"password"`
FetchViaProxy bool `json:"fetch_via_proxy"`
}
type feedModification struct {

@ -23,6 +23,7 @@ func (h *handler) getSubscriptions(w http.ResponseWriter, r *http.Request) {
subscriptionInfo.UserAgent,
subscriptionInfo.Username,
subscriptionInfo.Password,
subscriptionInfo.FetchViaProxy,
)
if finderErr != nil {
json.ServerError(w, r, finderErr)

@ -50,6 +50,7 @@ const (
defaultPocketConsumerKey = ""
defaultHTTPClientTimeout = 20
defaultHTTPClientMaxBodySize = 15
defaultHTTPClientProxy = ""
defaultAuthProxyHeader = ""
defaultAuthProxyUserCreation = false
)
@ -96,6 +97,7 @@ type Options struct {
pocketConsumerKey string
httpClientTimeout int
httpClientMaxBodySize int64
httpClientProxy string
authProxyHeader string
authProxyUserCreation bool
}
@ -141,6 +143,7 @@ func NewOptions() *Options {
pocketConsumerKey: defaultPocketConsumerKey,
httpClientTimeout: defaultHTTPClientTimeout,
httpClientMaxBodySize: defaultHTTPClientMaxBodySize * 1024 * 1024,
httpClientProxy: defaultHTTPClientProxy,
authProxyHeader: defaultAuthProxyHeader,
authProxyUserCreation: defaultAuthProxyUserCreation,
}
@ -349,6 +352,16 @@ func (o *Options) HTTPClientMaxBodySize() int64 {
return o.httpClientMaxBodySize
}
// HTTPClientProxy returns the proxy URL for HTTP client.
func (o *Options) HTTPClientProxy() string {
return o.httpClientProxy
}
// HasHTTPClientProxyConfigured returns true if the HTTP proxy is configured.
func (o *Options) HasHTTPClientProxyConfigured() bool {
return o.httpClientProxy != ""
}
// AuthProxyHeader returns an HTTP header name that contains username for
// authentication using auth proxy.
func (o *Options) AuthProxyHeader() string {
@ -403,6 +416,7 @@ func (o *Options) String() string {
builder.WriteString(fmt.Sprintf("OAUTH2_PROVIDER: %v\n", o.oauth2Provider))
builder.WriteString(fmt.Sprintf("HTTP_CLIENT_TIMEOUT: %v\n", o.httpClientTimeout))
builder.WriteString(fmt.Sprintf("HTTP_CLIENT_MAX_BODY_SIZE: %v\n", o.httpClientMaxBodySize))
builder.WriteString(fmt.Sprintf("HTTP_CLIENT_PROXY: %v\n", o.httpClientProxy))
builder.WriteString(fmt.Sprintf("AUTH_PROXY_HEADER: %v\n", o.authProxyHeader))
builder.WriteString(fmt.Sprintf("AUTH_PROXY_USER_CREATION: %v\n", o.authProxyUserCreation))
return builder.String()

@ -184,6 +184,8 @@ func (p *Parser) parseLines(lines []string) (err error) {
p.opts.httpClientTimeout = parseInt(value, defaultHTTPClientTimeout)
case "HTTP_CLIENT_MAX_BODY_SIZE":
p.opts.httpClientMaxBodySize = int64(parseInt(value, defaultHTTPClientMaxBodySize) * 1024 * 1024)
case "HTTP_CLIENT_PROXY":
p.opts.httpClientProxy = parseString(value, defaultHTTPClientProxy)
case "AUTH_PROXY_HEADER":
p.opts.authProxyHeader = parseString(value, defaultAuthProxyHeader)
case "AUTH_PROXY_USER_CREATION":

@ -12,7 +12,7 @@ import (
"miniflux.app/logger"
)
const schemaVersion = 34
const schemaVersion = 35
// Migrate executes database migrations.
func Migrate(db *sql.DB) {

@ -187,6 +187,8 @@ create index entries_user_feed_idx on entries (user_id, feed_id);
`,
"schema_version_33": `alter table users add column show_reading_time boolean default 't';`,
"schema_version_34": `CREATE INDEX entries_id_user_status_idx ON entries USING btree (id, user_id, status);`,
"schema_version_35": `alter table feeds add column fetch_via_proxy bool default false;
`,
"schema_version_4": `create type entry_sorting_direction as enum('asc', 'desc');
alter table users add column entry_direction entry_sorting_direction default 'asc';
`,
@ -244,6 +246,7 @@ var SqlMapChecksums = map[string]string{
"schema_version_32": "5b4de8dd2d7e3c6ae4150e0e3931df2ee989f2c667145bd67294e5a5f3fae456",
"schema_version_33": "bf38514efeb6c12511f41b1cc484f92722240b0a6ae874c32a958dfea3433d02",
"schema_version_34": "1a3e036f652fc98b7564a27013f04e1eb36dd0d68893c723168f134dc1065822",
"schema_version_35": "162a55df78eed4b9c9c141878132d5f1d97944b96f35a79e38f55716cdd6b3d2",
"schema_version_4": "216ea3a7d3e1704e40c797b5dc47456517c27dbb6ca98bf88812f4f63d74b5d9",
"schema_version_5": "46397e2f5f2c82116786127e9f6a403e975b14d2ca7b652a48cd1ba843e6a27c",
"schema_version_6": "9d05b4fb223f0e60efc716add5048b0ca9c37511cf2041721e20505d6d798ce4",

@ -0,0 +1 @@
alter table feeds add column fetch_via_proxy bool default false;

@ -47,6 +47,7 @@ type Client struct {
password string
userAgent string
Insecure bool
fetchViaProxy bool
}
func (c *Client) String() string {
@ -93,6 +94,12 @@ func (c *Client) WithCacheHeaders(etagHeader, lastModifiedHeader string) *Client
return c
}
// WithProxy enable proxy for current HTTP client request.
func (c *Client) WithProxy() *Client {
c.fetchViaProxy = true
return c
}
// WithUserAgent defines the User-Agent header to use for outgoing requests.
func (c *Client) WithUserAgent(userAgent string) *Client {
if userAgent != "" {
@ -230,12 +237,23 @@ func (c *Client) buildRequest(method string, body io.Reader) (*http.Request, err
func (c *Client) buildClient() http.Client {
client := http.Client{Timeout: time.Duration(config.Opts.HTTPClientTimeout()) * time.Second}
transport := &http.Transport{}
if c.Insecure {
client.Transport = &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
}
if c.fetchViaProxy && config.Opts.HasHTTPClientProxyConfigured() {
proxyURL, err := url.Parse(config.Opts.HTTPClientProxy())
if err != nil {
logger.Error("[HttpClient] Proxy URL error: %v", err)
} else {
logger.Debug("[HttpClient] Use proxy: %s", proxyURL)
transport.Proxy = http.ProxyURL(proxyURL)
}
}
client.Transport = transport
return client
}

@ -254,6 +254,7 @@ var translations = map[string]string{
"form.feed.label.scraper_rules": "Extraktionsregeln",
"form.feed.label.rewrite_rules": "Umschreiberegeln",
"form.feed.label.ignore_http_cache": "Ignoriere HTTP-cache",
"form.feed.label.fetch_via_proxy": "Über Proxy abrufen",
"form.feed.label.disabled": "Dieses Abonnement nicht aktualisieren",
"form.category.label.title": "Titel",
"form.user.label.username": "Benutzername",
@ -599,6 +600,7 @@ var translations = map[string]string{
"form.feed.label.scraper_rules": "Scraper Rules",
"form.feed.label.rewrite_rules": "Rewrite Rules",
"form.feed.label.ignore_http_cache": "Ignore HTTP cache",
"form.feed.label.fetch_via_proxy": "Fetch via proxy",
"form.feed.label.disabled": "Do not refresh this feed",
"form.category.label.title": "Title",
"form.user.label.username": "Username",
@ -924,6 +926,7 @@ var translations = map[string]string{
"form.feed.label.scraper_rules": "Reglas de raspador",
"form.feed.label.rewrite_rules": "Reglas de reescribir",
"form.feed.label.ignore_http_cache": "Ignorar caché HTTP",
"form.feed.label.fetch_via_proxy": "Buscar a través de proxy",
"form.feed.label.disabled": "No actualice este feed",
"form.category.label.title": "Título",
"form.user.label.username": "Nombre de usuario",
@ -1249,6 +1252,7 @@ var translations = map[string]string{
"form.feed.label.scraper_rules": "Règles pour récupérer le contenu original",
"form.feed.label.rewrite_rules": "Règles de réécriture",
"form.feed.label.ignore_http_cache": "Ignore cache HTTP",
"form.feed.label.fetch_via_proxy": "Récupérer via proxy",
"form.feed.label.disabled": "Ne pas actualiser ce flux",
"form.category.label.title": "Titre",
"form.user.label.username": "Nom d'utilisateur",
@ -1594,6 +1598,7 @@ var translations = map[string]string{
"form.feed.label.scraper_rules": "Regole di estrazione del contenuto",
"form.feed.label.rewrite_rules": "Regole di impaginazione del contenuto",
"form.feed.label.ignore_http_cache": "Ignora cache HTTP",
"form.feed.label.fetch_via_proxy": "Recuperare tramite proxy",
"form.feed.label.disabled": "Non aggiornare questo feed",
"form.category.label.title": "Titolo",
"form.user.label.username": "Nome utente",
@ -1919,6 +1924,7 @@ var translations = map[string]string{
"form.feed.label.scraper_rules": "スクラップルール",
"form.feed.label.rewrite_rules": "Rewrite ルール",
"form.feed.label.ignore_http_cache": "HTTPキャッシュを無視",
"form.feed.label.fetch_via_proxy": "プロキシ経由でフェッチ",
"form.feed.label.disabled": "このフィードを更新しない",
"form.category.label.title": "タイトル",
"form.user.label.username": "ユーザー名",
@ -2244,6 +2250,7 @@ var translations = map[string]string{
"form.feed.label.scraper_rules": "Scraper regels",
"form.feed.label.rewrite_rules": "Rewrite regels",
"form.feed.label.ignore_http_cache": "Negeer HTTP-cache",
"form.feed.label.fetch_via_proxy": "Ophalen via proxy",
"form.feed.label.disabled": "Vernieuw deze feed niet",
"form.category.label.title": "Naam",
"form.user.label.username": "Gebruikersnaam",
@ -2589,6 +2596,7 @@ var translations = map[string]string{
"form.feed.label.scraper_rules": "Zasady ekstrakcji",
"form.feed.label.rewrite_rules": "Reguły zapisu",
"form.feed.label.ignore_http_cache": "Zignoruj pamięć podręczną HTTP",
"form.feed.label.fetch_via_proxy": "Pobierz przez proxy",
"form.feed.label.disabled": "Не обновлять этот канал",
"form.category.label.title": "Tytuł",
"form.user.label.username": "Nazwa użytkownika",
@ -2939,6 +2947,7 @@ var translations = map[string]string{
"form.feed.label.rewrite_rules": "Regras para o Rewrite",
"form.feed.label.ignore_http_cache": "Ignorar cache HTTP",
"form.feed.label.disabled": "Não atualizar esta fonte",
"form.feed.label.fetch_via_proxy": "Buscar via proxy",
"form.category.label.title": "Título",
"form.user.label.username": "Nome de usuário",
"form.user.label.password": "Senha",
@ -3265,6 +3274,7 @@ var translations = map[string]string{
"form.feed.label.scraper_rules": "Правила Scraper",
"form.feed.label.rewrite_rules": "Правила Rewrite",
"form.feed.label.ignore_http_cache": "Игнорировать HTTP-кеш",
"form.feed.label.fetch_via_proxy": "Получить через прокси",
"form.feed.label.disabled": "Не обновлять этот канал",
"form.category.label.title": "Название",
"form.user.label.username": "Имя пользователя",
@ -3594,6 +3604,7 @@ var translations = map[string]string{
"form.feed.label.scraper_rules": "Scraper 规则",
"form.feed.label.rewrite_rules": "重写规则",
"form.feed.label.ignore_http_cache": "忽略HTTP缓存",
"form.feed.label.fetch_via_proxy": "通过代理获取",
"form.feed.label.disabled": "请勿刷新此Feed",
"form.category.label.title": "标题",
"form.user.label.username": "用户名",
@ -3684,15 +3695,15 @@ var translations = map[string]string{
}
var translationsChecksums = map[string]string{
"de_DE": "21e1bfb0f43d71efe38812b4337ddf6980c11ed18f4d06446ff7eda9dfa6b1f1",
"en_US": "30cbcb2170782f1e66f69066947bf053f68065d7b270eea879f2c573819dd52b",
"es_ES": "50dc7c8c2db7368bae133f5b455721470d314321153d41e4f27436a0f3f176e6",
"fr_FR": "373fd2db868961758bd1483c34f117b03aadea17080f268bc8bbd0acdfbc5eed",
"it_IT": "8d8f0bd75b4e7dec9370647c888dd9438b691130d9c41f839cdfff8cbc606cb5",
"ja_JP": "ec3a21c547e4625ad359624e43ba31b556fb8d8b8ff7fc7a20df089317db99b3",
"nl_NL": "20e180be2375f07ec02eb05f372a9102c13037a79e5651ce9bd41507fd2180d2",
"pl_PL": "b1526955641823708b4c1ca753b61e1e0561d0a3d33da3f62170540903031b0d",
"pt_BR": "cf8e131d39daac82d3157c6538c0643392a06358b7bc98be8579412ebd63f60e",
"ru_RU": "4056e4e94861835d44064273371adbbded7190e2b719769886eb99e6c9feaf82",
"zh_CN": "044abb0a34eee3d8d5597811d40166762311b8e4cd08b891796113790cc775f0",
"de_DE": "8f96cb46f5a7e8f64ee8f10176dc3a2f3d53953d250317da83a79d0700b47c82",
"en_US": "d33324caed406ecf6ce03920b15e235d46b258457a8bd48cd1ade685b9a3ad6b",
"es_ES": "2ff9333218dba2b86cb84f377dad66b9dc73848aee6bb09889cbdc10e58ca077",
"fr_FR": "07dc2cfdbc14cdf16312423158656f5526d3c3c7be490abf5503109a408e5056",
"it_IT": "d4f68a507e1deb9fab3aa38fb78d9e9e4040386d6f36611ec5f105adfb4b0d03",
"ja_JP": "5c4c063ebaee14bed941b020e0d19de5ef5e8d3bf11c1967b1f321d57d5af6a9",
"nl_NL": "f862027e192be7a09730470acc2639971c4abf01b256d5bb81246960cc54adcf",
"pl_PL": "35147e55f1800964d268dc04b9cc25a9c8fa98077f759c5d3a5bd339f0eee53e",
"pt_BR": "2461105ebc2a2d57b3a63a29ee21f74e3d1eba54c049abcfd077dd30acc8d0a2",
"ru_RU": "402f15d3c68e008a1ffa3dddb4002126a29c1cf93359a64bca944a8da541da72",
"zh_CN": "bbea61a08bec37518d8c8c7735c7b8001d011a1d43ddb15ab639fee11b45ca87",
}

@ -249,6 +249,7 @@
"form.feed.label.scraper_rules": "Extraktionsregeln",
"form.feed.label.rewrite_rules": "Umschreiberegeln",
"form.feed.label.ignore_http_cache": "Ignoriere HTTP-cache",
"form.feed.label.fetch_via_proxy": "Über Proxy abrufen",
"form.feed.label.disabled": "Dieses Abonnement nicht aktualisieren",
"form.category.label.title": "Titel",
"form.user.label.username": "Benutzername",

@ -249,6 +249,7 @@
"form.feed.label.scraper_rules": "Scraper Rules",
"form.feed.label.rewrite_rules": "Rewrite Rules",
"form.feed.label.ignore_http_cache": "Ignore HTTP cache",
"form.feed.label.fetch_via_proxy": "Fetch via proxy",
"form.feed.label.disabled": "Do not refresh this feed",
"form.category.label.title": "Title",
"form.user.label.username": "Username",

@ -249,6 +249,7 @@
"form.feed.label.scraper_rules": "Reglas de raspador",
"form.feed.label.rewrite_rules": "Reglas de reescribir",
"form.feed.label.ignore_http_cache": "Ignorar caché HTTP",
"form.feed.label.fetch_via_proxy": "Buscar a través de proxy",
"form.feed.label.disabled": "No actualice este feed",
"form.category.label.title": "Título",
"form.user.label.username": "Nombre de usuario",

@ -249,6 +249,7 @@
"form.feed.label.scraper_rules": "Règles pour récupérer le contenu original",
"form.feed.label.rewrite_rules": "Règles de réécriture",
"form.feed.label.ignore_http_cache": "Ignore cache HTTP",
"form.feed.label.fetch_via_proxy": "Récupérer via proxy",
"form.feed.label.disabled": "Ne pas actualiser ce flux",
"form.category.label.title": "Titre",
"form.user.label.username": "Nom d'utilisateur",

@ -249,6 +249,7 @@
"form.feed.label.scraper_rules": "Regole di estrazione del contenuto",
"form.feed.label.rewrite_rules": "Regole di impaginazione del contenuto",
"form.feed.label.ignore_http_cache": "Ignora cache HTTP",
"form.feed.label.fetch_via_proxy": "Recuperare tramite proxy",
"form.feed.label.disabled": "Non aggiornare questo feed",
"form.category.label.title": "Titolo",
"form.user.label.username": "Nome utente",

@ -249,6 +249,7 @@
"form.feed.label.scraper_rules": "スクラップルール",
"form.feed.label.rewrite_rules": "Rewrite ルール",
"form.feed.label.ignore_http_cache": "HTTPキャッシュを無視",
"form.feed.label.fetch_via_proxy": "プロキシ経由でフェッチ",
"form.feed.label.disabled": "このフィードを更新しない",
"form.category.label.title": "タイトル",
"form.user.label.username": "ユーザー名",

@ -249,6 +249,7 @@
"form.feed.label.scraper_rules": "Scraper regels",
"form.feed.label.rewrite_rules": "Rewrite regels",
"form.feed.label.ignore_http_cache": "Negeer HTTP-cache",
"form.feed.label.fetch_via_proxy": "Ophalen via proxy",
"form.feed.label.disabled": "Vernieuw deze feed niet",
"form.category.label.title": "Naam",
"form.user.label.username": "Gebruikersnaam",

@ -251,6 +251,7 @@
"form.feed.label.scraper_rules": "Zasady ekstrakcji",
"form.feed.label.rewrite_rules": "Reguły zapisu",
"form.feed.label.ignore_http_cache": "Zignoruj pamięć podręczną HTTP",
"form.feed.label.fetch_via_proxy": "Pobierz przez proxy",
"form.feed.label.disabled": "Не обновлять этот канал",
"form.category.label.title": "Tytuł",
"form.user.label.username": "Nazwa użytkownika",

@ -250,6 +250,7 @@
"form.feed.label.rewrite_rules": "Regras para o Rewrite",
"form.feed.label.ignore_http_cache": "Ignorar cache HTTP",
"form.feed.label.disabled": "Não atualizar esta fonte",
"form.feed.label.fetch_via_proxy": "Buscar via proxy",
"form.category.label.title": "Título",
"form.user.label.username": "Nome de usuário",
"form.user.label.password": "Senha",

@ -251,6 +251,7 @@
"form.feed.label.scraper_rules": "Правила Scraper",
"form.feed.label.rewrite_rules": "Правила Rewrite",
"form.feed.label.ignore_http_cache": "Игнорировать HTTP-кеш",
"form.feed.label.fetch_via_proxy": "Получить через прокси",
"form.feed.label.disabled": "Не обновлять этот канал",
"form.category.label.title": "Название",
"form.user.label.username": "Имя пользователя",

@ -247,6 +247,7 @@
"form.feed.label.scraper_rules": "Scraper 规则",
"form.feed.label.rewrite_rules": "重写规则",
"form.feed.label.ignore_http_cache": "忽略HTTP缓存",
"form.feed.label.fetch_via_proxy": "通过代理获取",
"form.feed.label.disabled": "请勿刷新此Feed",
"form.category.label.title": "标题",
"form.user.label.username": "用户名",

@ -250,6 +250,11 @@ Maximum body size for HTTP requests in Mebibyte (MiB)\&.
.br
Default is 15 MiB\&.
.TP
.B HTTP_CLIENT_PROXY
Proxy URL for HTTP client\&.
.br
Default is empty\&.
.TP
.B AUTH_PROXY_HEADER
Proxy authentication HTTP header\&.
.TP

@ -34,6 +34,7 @@ type Feed struct {
Password string `json:"password"`
Disabled bool `json:"disabled"`
IgnoreHTTPCache bool `json:"ignore_http_cache"`
FetchViaProxy bool `json:"fetch_via_proxy"`
Category *Category `json:"category,omitempty"`
Entries Entries `json:"entries,omitempty"`
Icon *FeedIcon `json:"icon"`
@ -71,13 +72,14 @@ func (f *Feed) WithCategoryID(categoryID int64) {
}
// WithBrowsingParameters defines browsing parameters.
func (f *Feed) WithBrowsingParameters(crawler bool, userAgent, username, password, scraperRules, rewriteRules string) {
func (f *Feed) WithBrowsingParameters(crawler bool, userAgent, username, password, scraperRules, rewriteRules string, fetchViaProxy bool) {
f.Crawler = crawler
f.UserAgent = userAgent
f.Username = username
f.Password = password
f.ScraperRules = scraperRules
f.RewriteRules = rewriteRules
f.FetchViaProxy = fetchViaProxy
}
// WithError adds a new error message and increment the error counter.

@ -48,7 +48,7 @@ func TestFeedCategorySetter(t *testing.T) {
func TestFeedBrowsingParams(t *testing.T) {
feed := &Feed{}
feed.WithBrowsingParameters(true, "Custom User Agent", "Username", "Secret", "Some Rule", "Another Rule")
feed.WithBrowsingParameters(true, "Custom User Agent", "Username", "Secret", "Some Rule", "Another Rule", false)
if !feed.Crawler {
t.Error(`The crawler must be activated`)

@ -34,7 +34,7 @@ type Handler struct {
}
// CreateFeed fetch, parse and store a new feed.
func (h *Handler) CreateFeed(userID, categoryID int64, url string, crawler bool, userAgent, username, password, scraperRules, rewriteRules string) (*model.Feed, error) {
func (h *Handler) CreateFeed(userID, categoryID int64, url string, crawler bool, userAgent, username, password, scraperRules, rewriteRules string, fetchViaProxy bool) (*model.Feed, error) {
defer timer.ExecutionTime(time.Now(), fmt.Sprintf("[Handler:CreateFeed] feedUrl=%s", url))
if !h.store.CategoryExists(userID, categoryID) {
@ -44,6 +44,11 @@ func (h *Handler) CreateFeed(userID, categoryID int64, url string, crawler bool,
request := client.New(url)
request.WithCredentials(username, password)
request.WithUserAgent(userAgent)
if fetchViaProxy {
request.WithProxy()
}
response, requestErr := browser.Exec(request)
if requestErr != nil {
return nil, requestErr
@ -60,7 +65,7 @@ func (h *Handler) CreateFeed(userID, categoryID int64, url string, crawler bool,
subscription.UserID = userID
subscription.WithCategoryID(categoryID)
subscription.WithBrowsingParameters(crawler, userAgent, username, password, scraperRules, rewriteRules)
subscription.WithBrowsingParameters(crawler, userAgent, username, password, scraperRules, rewriteRules, fetchViaProxy)
subscription.WithClientResponse(response)
subscription.CheckedNow()
@ -72,7 +77,7 @@ func (h *Handler) CreateFeed(userID, categoryID int64, url string, crawler bool,
logger.Debug("[Handler:CreateFeed] Feed saved with ID: %d", subscription.ID)
checkFeedIcon(h.store, subscription.ID, subscription.SiteURL)
checkFeedIcon(h.store, subscription.ID, subscription.SiteURL, fetchViaProxy)
return subscription, nil
}
@ -111,6 +116,10 @@ func (h *Handler) RefreshFeed(userID, feedID int64) error {
request.WithCacheHeaders(originalFeed.EtagHeader, originalFeed.LastModifiedHeader)
}
if originalFeed.FetchViaProxy {
request.WithProxy()
}
response, requestErr := browser.Exec(request)
if requestErr != nil {
originalFeed.WithError(requestErr.Localize(printer))
@ -141,7 +150,7 @@ func (h *Handler) RefreshFeed(userID, feedID int64) error {
// We update caching headers only if the feed has been modified,
// because some websites don't return the same headers when replying with a 304.
originalFeed.WithClientResponse(response)
checkFeedIcon(h.store, originalFeed.ID, originalFeed.SiteURL)
checkFeedIcon(h.store, originalFeed.ID, originalFeed.SiteURL, originalFeed.FetchViaProxy)
} else {
logger.Debug("[Handler:RefreshFeed] Feed #%d not modified", feedID)
}
@ -162,9 +171,9 @@ func NewFeedHandler(store *storage.Storage) *Handler {
return &Handler{store}
}
func checkFeedIcon(store *storage.Storage, feedID int64, websiteURL string) {
func checkFeedIcon(store *storage.Storage, feedID int64, websiteURL string, fetchViaProxy bool) {
if !store.HasIcon(feedID) {
icon, err := icon.FindIcon(websiteURL)
icon, err := icon.FindIcon(websiteURL, fetchViaProxy)
if err != nil {
logger.Debug("CheckFeedIcon: %v (feedID=%d websiteURL=%s)", err, feedID, websiteURL)
} else if icon == nil {

@ -21,9 +21,12 @@ import (
)
// FindIcon try to find the website's icon.
func FindIcon(websiteURL string) (*model.Icon, error) {
func FindIcon(websiteURL string, fetchViaProxy bool) (*model.Icon, error) {
rootURL := url.RootURL(websiteURL)
clt := client.New(rootURL)
if fetchViaProxy {
clt.WithProxy()
}
response, err := clt.Get()
if err != nil {
return nil, fmt.Errorf("unable to download website index page: %v", err)
@ -43,7 +46,7 @@ func FindIcon(websiteURL string) (*model.Icon, error) {
}
logger.Debug("[FindIcon] Fetching icon => %s", iconURL)
icon, err := downloadIcon(iconURL)
icon, err := downloadIcon(iconURL, fetchViaProxy)
if err != nil {
return nil, err
}
@ -86,8 +89,11 @@ func parseDocument(websiteURL string, data io.Reader) (string, error) {
return iconURL, nil
}
func downloadIcon(iconURL string) (*model.Icon, error) {
func downloadIcon(iconURL string, fetchViaProxy bool) (*model.Icon, error) {
clt := client.New(iconURL)
if fetchViaProxy {
clt.WithProxy()
}
response, err := clt.Get()
if err != nil {
return nil, fmt.Errorf("unable to download iconURL: %v", err)

@ -26,13 +26,18 @@ var (
)
// FindSubscriptions downloads and try to find one or more subscriptions from an URL.
func FindSubscriptions(websiteURL, userAgent, username, password string) (Subscriptions, *errors.LocalizedError) {
func FindSubscriptions(websiteURL, userAgent, username, password string, fetchViaProxy bool) (Subscriptions, *errors.LocalizedError) {
websiteURL = findYoutubeChannelFeed(websiteURL)
websiteURL = parseYoutubeVideoPage(websiteURL)
request := client.New(websiteURL)
request.WithCredentials(username, password)
request.WithUserAgent(userAgent)
if fetchViaProxy {
request.WithProxy()
}
response, err := browser.Exec(request)
if err != nil {
return nil, err

@ -32,6 +32,7 @@ var feedListQuery = `
f.username,
f.password,
f.ignore_http_cache,
f.fetch_via_proxy,
f.disabled,
f.category_id,
c.title as category_title,
@ -133,6 +134,7 @@ func (s *Storage) FeedsByCategoryWithCounters(userID, categoryID int64) (model.F
f.username,
f.password,
f.ignore_http_cache,
f.fetch_via_proxy,
f.disabled,
f.category_id,
c.title as category_title,
@ -242,6 +244,7 @@ func (s *Storage) fetchFeeds(feedQuery, counterQuery string, args ...interface{}
&feed.Username,
&feed.Password,
&feed.IgnoreHTTPCache,
&feed.FetchViaProxy,
&feed.Disabled,
&feed.Category.ID,
&feed.Category.Title,
@ -326,6 +329,7 @@ func (s *Storage) FeedByID(userID, feedID int64) (*model.Feed, error) {
f.username,
f.password,
f.ignore_http_cache,
f.fetch_via_proxy,
f.disabled,
f.category_id,
c.title as category_title,
@ -357,6 +361,7 @@ func (s *Storage) FeedByID(userID, feedID int64) (*model.Feed, error) {
&feed.Username,
&feed.Password,
&feed.IgnoreHTTPCache,
&feed.FetchViaProxy,
&feed.Disabled,
&feed.Category.ID,
&feed.Category.Title,
@ -396,10 +401,11 @@ func (s *Storage) CreateFeed(feed *model.Feed) error {
password,
disabled,
scraper_rules,
rewrite_rules
rewrite_rules,
fetch_via_proxy
)
VALUES
($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)
($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15)
RETURNING
id
`
@ -419,6 +425,7 @@ func (s *Storage) CreateFeed(feed *model.Feed) error {
feed.Disabled,
feed.ScraperRules,
feed.RewriteRules,
feed.FetchViaProxy,
).Scan(&feed.ID)
if err != nil {
return fmt.Errorf(`store: unable to create feed %q: %v`, feed.FeedURL, err)
@ -462,9 +469,10 @@ func (s *Storage) UpdateFeed(feed *model.Feed) (err error) {
password=$15,
disabled=$16,
next_check_at=$17,
ignore_http_cache=$18
ignore_http_cache=$18,
fetch_via_proxy=$19
WHERE
id=$19 AND user_id=$20
id=$20 AND user_id=$21
`
_, err = s.db.Exec(query,
feed.FeedURL,
@ -485,6 +493,7 @@ func (s *Storage) UpdateFeed(feed *model.Feed) (err error) {
feed.Disabled,
feed.NextCheckAt,
feed.IgnoreHTTPCache,
feed.FetchViaProxy,
feed.ID,
feed.UserID,
)

@ -519,7 +519,7 @@ SOFTWARE.
var templateCommonMapChecksums = map[string]string{
"entry_pagination": "cdca9cf12586e41e5355190b06d9168f57f77b85924d1e63b13524bc15abcbf6",
"feed_list": "30acc9ecc413811e73a1dad120b5d44e29564de3ba794fb07ee886b30addfb19",
"feed_list": "931e43d328a116318c510de5658c688cd940b934c86b6ec82a472e1f81e020ae",
"feed_menu": "318d8662dda5ca9dfc75b909c8461e79c86fb5082df1428f67aaf856f19f4b50",
"icons": "3dbe754a98f524a227111191d76b8c6944711b13613cc548ee9e9808fe0bffb4",
"item_meta": "8306adf3ef9966de3e3dc74ca1042e51d778b027ab8cf0a60a2e94a0115982dc",

@ -30,6 +30,9 @@
<summary>{{ t "page.add_feed.legend.advanced_options" }}</summary>
<div class="details-content">
<label><input type="checkbox" name="crawler" value="1" {{ if .form.Crawler }}checked{{ end }}> {{ t "form.feed.label.crawler" }}</label>
{{ if .hasProxyConfigured }}
<label><input type="checkbox" name="fetch_via_proxy" value="1" {{ if .form.FetchViaProxy }}checked{{ end }}> {{ t "form.feed.label.fetch_via_proxy" }}</label>
{{ end }}
<label for="form-user-agent">{{ t "form.feed.label.user_agent" }}</label>
<input type="text" name="user_agent" id="form-user-agent" placeholder="{{ .defaultUserAgent }}" value="{{ .form.UserAgent }}" autocomplete="off">

@ -14,6 +14,9 @@
<input type="hidden" name="feed_password" value="{{ .form.Password }}">
<input type="hidden" name="scraper_rules" value="{{ .form.ScraperRules }}">
<input type="hidden" name="rewrite_rules" value="{{ .form.RewriteRules }}">
{{ if .form.FetchViaProxy }}
<input type="hidden" name="fetch_via_proxy" value="1">
{{ end }}
{{ if .form.Crawler }}
<input type="hidden" name="crawler" value="1">
{{ end }}

@ -73,6 +73,9 @@
<label><input type="checkbox" name="crawler" value="1" {{ if .form.Crawler }}checked{{ end }}> {{ t "form.feed.label.crawler" }}</label>
<label><input type="checkbox" name="ignore_http_cache" value="1" {{ if .form.IgnoreHTTPCache }}checked{{ end }}> {{ t "form.feed.label.ignore_http_cache" }}</label>
{{ if .hasProxyConfigured }}
<label><input type="checkbox" name="fetch_via_proxy" value="1" {{ if .form.FetchViaProxy }}checked{{ end }}> {{ t "form.feed.label.fetch_via_proxy" }}</label>
{{ end }}
<label><input type="checkbox" name="disabled" value="1" {{ if .form.Disabled }}checked{{ end }}> {{ t "form.feed.label.disabled" }}</label>
<div class="buttons">

@ -61,6 +61,9 @@ var templateViewsMap = map[string]string{
<summary>{{ t "page.add_feed.legend.advanced_options" }}</summary>
<div class="details-content">
<label><input type="checkbox" name="crawler" value="1" {{ if .form.Crawler }}checked{{ end }}> {{ t "form.feed.label.crawler" }}</label>
{{ if .hasProxyConfigured }}
<label><input type="checkbox" name="fetch_via_proxy" value="1" {{ if .form.FetchViaProxy }}checked{{ end }}> {{ t "form.feed.label.fetch_via_proxy" }}</label>
{{ end }}
<label for="form-user-agent">{{ t "form.feed.label.user_agent" }}</label>
<input type="text" name="user_agent" id="form-user-agent" placeholder="{{ .defaultUserAgent }}" value="{{ .form.UserAgent }}" autocomplete="off">
@ -380,6 +383,9 @@ var templateViewsMap = map[string]string{
<input type="hidden" name="feed_password" value="{{ .form.Password }}">
<input type="hidden" name="scraper_rules" value="{{ .form.ScraperRules }}">
<input type="hidden" name="rewrite_rules" value="{{ .form.RewriteRules }}">
{{ if .form.FetchViaProxy }}
<input type="hidden" name="fetch_via_proxy" value="1">
{{ end }}
{{ if .form.Crawler }}
<input type="hidden" name="crawler" value="1">
{{ end }}
@ -592,6 +598,9 @@ var templateViewsMap = map[string]string{
<label><input type="checkbox" name="crawler" value="1" {{ if .form.Crawler }}checked{{ end }}> {{ t "form.feed.label.crawler" }}</label>
<label><input type="checkbox" name="ignore_http_cache" value="1" {{ if .form.IgnoreHTTPCache }}checked{{ end }}> {{ t "form.feed.label.ignore_http_cache" }}</label>
{{ if .hasProxyConfigured }}
<label><input type="checkbox" name="fetch_via_proxy" value="1" {{ if .form.FetchViaProxy }}checked{{ end }}> {{ t "form.feed.label.fetch_via_proxy" }}</label>
{{ end }}
<label><input type="checkbox" name="disabled" value="1" {{ if .form.Disabled }}checked{{ end }}> {{ t "form.feed.label.disabled" }}</label>
<div class="buttons">
@ -1548,18 +1557,18 @@ var templateViewsMap = map[string]string{
var templateViewsMapChecksums = map[string]string{
"about": "4035658497363d7af7f79be83190404eb21ec633fe8ec636bdfc219d9fc78cfc",
"add_subscription": "0dbea93b6fc07423fa066122ad960c69616b829533371a2dbadec1e22d4f1ae0",
"add_subscription": "63961a83964acca354bc30eaae1f5e80f410ae4091af8da317380d4298f79032",
"api_keys": "27d401b31a72881d5232486ba17eb47edaf5246eaedce81de88698c15ebb2284",
"bookmark_entries": "892fe6cbf5a3301416dfb76e62935b495ca194275cfe113105a85b40ce7c200f",
"categories": "9dfc3cb7bb91c7750753fe962ee4540dd1843e5f75f9e0a575ee964f6f9923e9",
"category_entries": "8fa0e0b8f85e2572c40dee855b6d636207c3561086b234c93100673774c06746",
"category_feeds": "07154127087f9b127f7290abad6020c35ad9ceb2490b869120b7628bc4413808",
"choose_subscription": "84c9730cadd78e6ee5a6b4c499aab33acddb4324ac01924d33387543eec4d702",
"choose_subscription": "22109d760ea8079c491561d0106f773c885efbf66f87d81fcf8700218260d2a0",
"create_api_key": "5f74d4e92a6684927f5305096378c8be278159a5cd88ce652c7be3280a7d1685",
"create_category": "6b22b5ce51abf4e225e23a79f81be09a7fb90acb265e93a8faf9446dff74018d",
"create_user": "9b73a55233615e461d1f07d99ad1d4d3b54532588ab960097ba3e090c85aaf3a",
"edit_category": "b1c0b38f1b714c5d884edcd61e5b5295a5f1c8b71c469b35391e4dcc97cc6d36",
"edit_feed": "ff90b1883e2934e0236d530d8b778affe7665a6b08cdf9ae612c7e56469818ef",
"edit_feed": "7e86275f8e9325ddbffe79f6db871e58ad86d08c396e9b2ff8af69a09c4bf63b",
"edit_user": "c692db9de1a084c57b93e95a14b041d39bf489846cbb91fc982a62b72b77062a",
"entry": "c503dcf77de37090b9f05352bb9d99729085eec6e7bc22be94f2b4b244b4e48c",
"feed_entries": "ea5b88e3ad6b166d83b70e021d7b420d025f80decb6e24c79d13f8ce7c910b04",

@ -7,6 +7,7 @@ package ui // import "miniflux.app/ui"
import (
"net/http"
"miniflux.app/config"
"miniflux.app/http/client"
"miniflux.app/http/request"
"miniflux.app/http/response/html"
@ -52,6 +53,7 @@ func (h *handler) showEditFeedPage(w http.ResponseWriter, r *http.Request) {
Username: feed.Username,
Password: feed.Password,
IgnoreHTTPCache: feed.IgnoreHTTPCache,
FetchViaProxy: feed.FetchViaProxy,
Disabled: feed.Disabled,
}
@ -65,6 +67,7 @@ func (h *handler) showEditFeedPage(w http.ResponseWriter, r *http.Request) {
view.Set("countUnread", h.store.CountUnreadEntries(user.ID))
view.Set("countErrorFeeds", h.store.CountErrorFeeds(user.ID))
view.Set("defaultUserAgent", client.DefaultUserAgent)
view.Set("hasProxyConfigured", config.Opts.HasHTTPClientProxyConfigured())
html.OK(w, r, view.Render("edit_feed"))
}

@ -25,6 +25,7 @@ type FeedForm struct {
Username string
Password string
IgnoreHTTPCache bool
FetchViaProxy bool
Disabled bool
}
@ -51,6 +52,7 @@ func (f FeedForm) Merge(feed *model.Feed) *model.Feed {
feed.Username = f.Username
feed.Password = f.Password
feed.IgnoreHTTPCache = f.IgnoreHTTPCache
feed.FetchViaProxy = f.FetchViaProxy
feed.Disabled = f.Disabled
return feed
}
@ -74,6 +76,7 @@ func NewFeedForm(r *http.Request) *FeedForm {
Username: r.FormValue("feed_username"),
Password: r.FormValue("feed_password"),
IgnoreHTTPCache: r.FormValue("ignore_http_cache") == "1",
FetchViaProxy: r.FormValue("fetch_via_proxy") == "1",
Disabled: r.FormValue("disabled") == "1",
}
}

@ -13,14 +13,15 @@ import (
// SubscriptionForm represents the subscription form.
type SubscriptionForm struct {
URL string
CategoryID int64
Crawler bool
UserAgent string
Username string
Password string
ScraperRules string
RewriteRules string
URL string
CategoryID int64
Crawler bool
FetchViaProxy bool
UserAgent string
Username string
Password string
ScraperRules string
RewriteRules string
}
// Validate makes sure the form values are valid.
@ -40,13 +41,14 @@ func NewSubscriptionForm(r *http.Request) *SubscriptionForm {
}
return &SubscriptionForm{
URL: r.FormValue("url"),
Crawler: r.FormValue("crawler") == "1",
CategoryID: int64(categoryID),
UserAgent: r.FormValue("user_agent"),
Username: r.FormValue("feed_username"),
Password: r.FormValue("feed_password"),
ScraperRules: r.FormValue("scraper_rules"),
RewriteRules: r.FormValue("rewrite_rules"),
URL: r.FormValue("url"),
Crawler: r.FormValue("crawler") == "1",
FetchViaProxy: r.FormValue("fetch_via_proxy") == "1",
CategoryID: int64(categoryID),
UserAgent: r.FormValue("user_agent"),
Username: r.FormValue("feed_username"),
Password: r.FormValue("feed_password"),
ScraperRules: r.FormValue("scraper_rules"),
RewriteRules: r.FormValue("rewrite_rules"),
}
}

@ -2,14 +2,15 @@
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package ui // import "miniflux.app/ui"
package ui // import "miniflux.app/ui"
import (
"net/http"
"miniflux.app/config"
"miniflux.app/http/client"
"miniflux.app/http/response/html"
"miniflux.app/http/request"
"miniflux.app/http/response/html"
"miniflux.app/ui/form"
"miniflux.app/ui/session"
"miniflux.app/ui/view"
@ -38,6 +39,7 @@ func (h *handler) showAddSubscriptionPage(w http.ResponseWriter, r *http.Request
view.Set("countErrorFeeds", h.store.CountErrorFeeds(user.ID))
view.Set("defaultUserAgent", client.DefaultUserAgent)
view.Set("form", &form.SubscriptionForm{CategoryID: 0})
view.Set("hasProxyConfigured", config.Opts.HasHTTPClientProxyConfigured())
html.OK(w, r, view.Render("add_subscription"))
}

@ -2,11 +2,12 @@
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package ui // import "miniflux.app/ui"
package ui // import "miniflux.app/ui"
import (
"net/http"
"miniflux.app/config"
"miniflux.app/http/client"
"miniflux.app/http/request"
"miniflux.app/http/response/html"
@ -40,6 +41,7 @@ func (h *handler) bookmarklet(w http.ResponseWriter, r *http.Request) {
view.Set("countUnread", h.store.CountUnreadEntries(user.ID))
view.Set("countErrorFeeds", h.store.CountErrorFeeds(user.ID))
view.Set("defaultUserAgent", client.DefaultUserAgent)
view.Set("hasProxyConfigured", config.Opts.HasHTTPClientProxyConfigured())
html.OK(w, r, view.Render("add_subscription"))
}

@ -57,6 +57,7 @@ func (h *handler) showChooseSubscriptionPage(w http.ResponseWriter, r *http.Requ
subscriptionForm.Password,
subscriptionForm.ScraperRules,
subscriptionForm.RewriteRules,
subscriptionForm.FetchViaProxy,
)
if err != nil {
view.Set("form", subscriptionForm)

@ -7,6 +7,7 @@ package ui // import "miniflux.app/ui"
import (
"net/http"
"miniflux.app/config"
"miniflux.app/http/client"
"miniflux.app/http/request"
"miniflux.app/http/response/html"
@ -40,6 +41,7 @@ func (h *handler) submitSubscription(w http.ResponseWriter, r *http.Request) {
v.Set("countUnread", h.store.CountUnreadEntries(user.ID))
v.Set("countErrorFeeds", h.store.CountErrorFeeds(user.ID))
v.Set("defaultUserAgent", client.DefaultUserAgent)
v.Set("hasProxyConfigured", config.Opts.HasHTTPClientProxyConfigured())
subscriptionForm := form.NewSubscriptionForm(r)
if err := subscriptionForm.Validate(); err != nil {
@ -54,6 +56,7 @@ func (h *handler) submitSubscription(w http.ResponseWriter, r *http.Request) {
subscriptionForm.UserAgent,
subscriptionForm.Username,
subscriptionForm.Password,
subscriptionForm.FetchViaProxy,
)
if findErr != nil {
logger.Error("[UI:SubmitSubscription] %s", findErr)
@ -82,6 +85,7 @@ func (h *handler) submitSubscription(w http.ResponseWriter, r *http.Request) {
subscriptionForm.Password,
subscriptionForm.ScraperRules,
subscriptionForm.RewriteRules,
subscriptionForm.FetchViaProxy,
)
if err != nil {
v.Set("form", subscriptionForm)
@ -99,6 +103,7 @@ func (h *handler) submitSubscription(w http.ResponseWriter, r *http.Request) {
v.Set("user", user)
v.Set("countUnread", h.store.CountUnreadEntries(user.ID))
v.Set("countErrorFeeds", h.store.CountErrorFeeds(user.ID))
v.Set("hasProxyConfigured", config.Opts.HasHTTPClientProxyConfigured())
html.OK(w, r, v.Render("choose_subscription"))
}

Loading…
Cancel
Save