Add option to use Reddit's onion service to avoid blcoks

.env.example

@@ -10,6 +10,10 @@ REDLIB_BANNER=
 REDLIB_ROBOTS_DISABLE_INDEXING=off
 # Set the Pushshift frontend for "removed" links
 REDLIB_PUSHSHIFT_FRONTEND=undelete.pullpush.io
+# Uses Reddit's onion service to avoid ratelimiting and other blocks. Requires ALL_PROXY set to a Tor socks5h proxy.
+REDLIB_USE_TOR=off
+# Set proxy for Redlib to use
+#ALL_PROXY=socks5h://127.0.0.1:9050
 
 # Default user settings
 # Set the default theme (options: system, light, dark, black, dracula, nord, laserwave, violet, gold, rosebox, gruvboxdark, gruvboxlight)

README.md

@@ -408,15 +408,16 @@ Redlib supports the following command line flags:
 
 Assign a default value for each instance-specific setting by passing environment variables to Redlib in the format `REDLIB_{X}`. Replace `{X}` with the setting name (see list below) in capital letters.
 
-| Name                      | Possible values | Default value          | Description                                                                                               |
-|---------------------------|-----------------|------------------------|-----------------------------------------------------------------------------------------------------------|
-| `SFW_ONLY`                | `["on", "off"]` | `off`                  | Enables SFW-only mode for the instance, i.e. all NSFW content is filtered.                                |
-| `BANNER`                  | String          | (empty)                | Allows the server to set a banner to be displayed. Currently this is displayed on the instance info page. |
-| `ROBOTS_DISABLE_INDEXING` | `["on", "off"]` | `off`                  | Disables indexing of the instance by search engines.                                                      |
-| `PUSHSHIFT_FRONTEND`      | String          | `undelete.pullpush.io` | Allows the server to set the Pushshift frontend to be used with "removed" links.                          |
-| `PORT`                    | Integer 0-65535 | `8080`                 | The **internal** port Redlib listens on.                                                                  |
-| `ENABLE_RSS`              | `["on", "off"]` | `off`                  | Enables RSS feed generation.                                                                              |
-| `FULL_URL`                | String          | (empty)                | Allows for proper URLs (for now, only needed by RSS)                                                      |
+| Name                      | Possible values | Default value          | Description                                                                                                        |
+|---------------------------|-----------------|------------------------|--------------------------------------------------------------------------------------------------------------------|
+| `SFW_ONLY`                | `["on", "off"]` | `off`                  | Enables SFW-only mode for the instance, i.e. all NSFW content is filtered.                                         |
+| `BANNER`                  | String          | (empty)                | Allows the server to set a banner to be displayed. Currently this is displayed on the instance info page.          |
+| `ROBOTS_DISABLE_INDEXING` | `["on", "off"]` | `off`                  | Disables indexing of the instance by search engines.                                                               |
+| `PUSHSHIFT_FRONTEND`      | String          | `undelete.pullpush.io` | Allows the server to set the Pushshift frontend to be used with "removed" links.                                   |
+| `PORT`                    | Integer 0-65535 | `8080`                 | The **internal** port Redlib listens on.                                                                           |
+| `ENABLE_RSS`              | `["on", "off"]` | `off`                  | Enables RSS feed generation.                                                                                       |
+| `FULL_URL`                | String          | (empty)                | Allows for proper URLs (for now, only needed by RSS)                                                               |
+| `USE_TOR`                 | String          | `off`                  | Uses Reddit's onion service to avoid ratelimiting and other blocks. Requires ALL_PROXY set to a Tor socks5h proxy. |
 
 ## Default user settings
 

src/client.rs

@@ -1,6 +1,7 @@
 use crate::dbg_msg;
 use crate::oauth::{force_refresh_token, token_daemon, Oauth, OauthBackendImpl};
 use crate::server::RequestExt;
+use crate::utils;
 use crate::utils::{format_url, Post};
 use arc_swap::ArcSwap;
 use cached::proc_macro::cached;
@@ -18,15 +19,6 @@ use wreq::redirect::Policy;
 use wreq::{header as wreq_header, Client as WreqClient, EmulationFactory, Method, Response as WreqResponse};
 use wreq_util::{Emulation, EmulationOS, EmulationOption};
 
-const REDDIT_URL_BASE: &str = "https://oauth.reddit.com";
-const REDDIT_URL_BASE_HOST: &str = "oauth.reddit.com";
-
-const REDDIT_SHORT_URL_BASE: &str = "https://redd.it";
-const REDDIT_SHORT_URL_BASE_HOST: &str = "redd.it";
-
-const ALTERNATIVE_REDDIT_URL_BASE: &str = "https://www.reddit.com";
-const ALTERNATIVE_REDDIT_URL_BASE_HOST: &str = "www.reddit.com";
-
 pub static CLIENT: LazyLock<WreqClient> = LazyLock::new(build_client);
 
 pub static OAUTH_CLIENT: LazyLock<ArcSwap<Oauth>> = LazyLock::new(|| {
@@ -39,10 +31,12 @@ pub static OAUTH_RATELIMIT_REMAINING: AtomicU16 = AtomicU16::new(99);
 
 pub static OAUTH_IS_ROLLING_OVER: AtomicBool = AtomicBool::new(false);
 
-const URL_PAIRS: [(&str, &str); 2] = [
-	(ALTERNATIVE_REDDIT_URL_BASE, ALTERNATIVE_REDDIT_URL_BASE_HOST),
-	(REDDIT_SHORT_URL_BASE, REDDIT_SHORT_URL_BASE_HOST),
-];
+pub fn url_pairs() -> [(&'static str, &'static str); 2] {
+	[
+		(utils::get_alternative_reddit_url_base(), utils::get_alternative_reddit_url_base_host()),
+		(utils::get_short_reddit_url_base(), utils::get_short_reddit_url_base_host()),
+	]
+}
 
 pub fn build_client() -> WreqClient {
 	// Keeping this list short to aid in privacy.
@@ -89,7 +83,7 @@ pub async fn canonical_path(path: String, tries: i8) -> Result<Option<String>, S
 	let res = {
 		// for url base and host in URL_PAIRS, try reddit_short_head(path.clone(), true, url_base, url_base_host) and if it succeeds, set res. else, res = None
 		let mut res = None;
-		for (url_base, url_base_host) in URL_PAIRS {
+		for (url_base, url_base_host) in url_pairs() {
 			res = reddit_short_head(path.clone(), true, url_base, url_base_host).await.ok();
 			if let Some(res) = &res {
 				if !res.status().is_client_error() {
@@ -148,7 +142,7 @@ pub async fn canonical_path(path: String, tries: i8) -> Result<Option<String>, S
 			res
 				.headers()
 				.get(wreq_header::LOCATION)
-				.map(|val| percent_encode(val.as_bytes(), CONTROLS).to_string().trim_start_matches(REDDIT_URL_BASE).to_string()),
+				.map(|val| percent_encode(val.as_bytes(), CONTROLS).to_string().trim_start_matches(utils::get_reddit_url_base()).to_string()),
 		),
 	}
 }
@@ -212,7 +206,7 @@ pub async fn proxy(req: HyperRequest<Body>, format: &str) -> Result<HyperRespons
 /// Makes a GET request to Reddit at `path`. By default, this will honor HTTP
 /// 3xx codes Reddit returns and will automatically redirect.
 fn reddit_get(path: String, quarantine: bool) -> Boxed<Result<WreqResponse, String>> {
-	request(&Method::GET, path, true, quarantine, REDDIT_URL_BASE, REDDIT_URL_BASE_HOST)
+	request(&Method::GET, path, true, quarantine, utils::get_reddit_url_base(), utils::get_reddit_url_base_host())
 }
 
 /// Makes a HEAD request to Reddit at `path, using the short URL base. This will not follow redirects.
@@ -271,7 +265,7 @@ fn request(method: &'static Method, path: String, redirect: bool, quarantine: bo
 						return Ok(response);
 					};
 					let location_header = response.headers().get(wreq::header::LOCATION);
-					if location_header.and_then(|h| h.to_str().ok()) == Some(ALTERNATIVE_REDDIT_URL_BASE) {
+					if location_header.and_then(|h| h.to_str().ok()) == Some(utils::get_alternative_reddit_url_base()) {
 						return Err("Reddit response was invalid".to_string());
 					}
 					return request(
@@ -291,8 +285,8 @@ fn request(method: &'static Method, path: String, redirect: bool, quarantine: bo
 								//     2. Percent-encode the path.
 								let new_path = percent_encode(val.as_bytes(), CONTROLS)
 									.to_string()
-									.trim_start_matches(REDDIT_URL_BASE)
-									.trim_start_matches(ALTERNATIVE_REDDIT_URL_BASE)
+									.trim_start_matches(utils::get_reddit_url_base())
+									.trim_start_matches(utils::get_alternative_reddit_url_base())
 									.to_string();
 								format!("{new_path}{}raw_json=1", if new_path.contains('?') { "&" } else { "?" })
 							})
@@ -309,7 +303,7 @@ fn request(method: &'static Method, path: String, redirect: bool, quarantine: bo
 				Ok(response)
 			}
 			Err(e) => {
-				dbg_msg!("{method} {REDDIT_URL_BASE}{path}: {}", e);
+				dbg_msg!("{method} {}{path}: {}", utils::get_reddit_url_base(), e);
 
 				Err(e.to_string())
 			}

src/config.rs

@@ -108,6 +108,9 @@ pub struct Config {
 
 	#[serde(rename = "REDLIB_DEFAULT_REMOVE_DEFAULT_FEEDS")]
 	pub(crate) default_remove_default_feeds: Option<String>,
+
+	#[serde(rename = "REDLIB_USE_TOR")]
+	pub(crate) use_tor: Option<String>,
 }
 
 impl Config {
@@ -156,6 +159,7 @@ impl Config {
 			enable_rss: parse("REDLIB_ENABLE_RSS"),
 			full_url: parse("REDLIB_FULL_URL"),
 			default_remove_default_feeds: parse("REDLIB_DEFAULT_REMOVE_DEFAULT_FEEDS"),
+			use_tor: parse("REDLIB_USE_TOR"),
 		}
 	}
 }
@@ -186,6 +190,7 @@ fn get_setting_from_config(name: &str, config: &Config) -> Option<String> {
 		"REDLIB_ENABLE_RSS" => config.enable_rss.clone(),
 		"REDLIB_FULL_URL" => config.full_url.clone(),
 		"REDLIB_DEFAULT_REMOVE_DEFAULT_FEEDS" => config.default_remove_default_feeds.clone(),
+		"REDLIB_USE_TOR" => config.use_tor.clone(),
 		_ => None,
 	}
 }

src/main.rs

@@ -262,7 +262,7 @@ async fn main() {
 	app.at("/instances.json").get(|_| async move { proxy_instances().await }.boxed());
 
 	// Proxy media through Redlib
-	app.at("/vid/:id/:size").get(|r| proxy(r, "https://v.redd.it/{id}/DASH_{size}").boxed());
+	app.at("/vid/:id/:prefix/:size").get(|r| proxy(r, "https://v.redd.it/{id}/{prefix}_{size}").boxed());
 	app.at("/hls/:id/*path").get(|r| proxy(r, "https://v.redd.it/{id}/{path}").boxed());
 	app.at("/img/*path").get(|r| proxy(r, "https://i.redd.it/{path}").boxed());
 	app.at("/thumb/:point/:id").get(|r| proxy(r, "https://{point}.thumbs.redditmedia.com/{id}").boxed());

src/oauth.rs

@@ -1,6 +1,7 @@
 use crate::{
 	client::{CLIENT, OAUTH_CLIENT, OAUTH_IS_ROLLING_OVER, OAUTH_RATELIMIT_REMAINING},
 	oauth_resources::ANDROID_APP_VERSION_LIST,
+	utils::{self, get_oauth_timeout},
 };
 use base64::{engine::general_purpose, Engine as _};
 use log::{error, info, trace, warn};
@@ -11,10 +12,6 @@ use tokio::time::{error::Elapsed, timeout};
 
 const REDDIT_ANDROID_OAUTH_CLIENT_ID: &str = "ohXpoqrZYub1kg";
 
-const AUTH_ENDPOINT: &str = "https://www.reddit.com";
-
-const OAUTH_TIMEOUT: Duration = Duration::from_secs(5);
-
 // Response from OAuth backend authentication
 #[derive(Debug, Clone)]
 pub struct OauthResponse {
@@ -111,12 +108,12 @@ impl Oauth {
 				std::process::exit(1);
 			}
 
-			tokio::time::sleep(OAUTH_TIMEOUT).await;
+			tokio::time::sleep(get_oauth_timeout()).await;
 		}
 	}
 
 	async fn new_with_timeout_with_backend(mut backend: OauthBackendImpl) -> Result<Result<Self, AuthError>, Elapsed> {
-		timeout(OAUTH_TIMEOUT, async move {
+		timeout(get_oauth_timeout(), async move {
 			let response = backend.authenticate().await?;
 
 			// Build headers_map from backend headers + Authorization header
@@ -219,7 +216,8 @@ impl MobileSpoofAuth {
 impl OauthBackend for MobileSpoofAuth {
 	async fn authenticate(&mut self) -> Result<OauthResponse, AuthError> {
 		// Construct URL for OAuth token
-		let url = format!("{AUTH_ENDPOINT}/auth/v2/oauth/access-token/loid");
+		let auth_endpoint = utils::get_alternative_reddit_url_base();
+		let url = format!("{auth_endpoint}/auth/v2/oauth/access-token/loid");
 		let mut builder = CLIENT.post(&url);
 
 		// Add headers from spoofed client
@@ -334,11 +332,12 @@ impl GenericWebAuth {
 impl OauthBackend for GenericWebAuth {
 	async fn authenticate(&mut self) -> Result<OauthResponse, AuthError> {
 		// Construct URL for OAuth token
-		let url = "https://www.reddit.com/api/v1/access_token";
-		let mut builder = CLIENT.post(url);
+		let auth_endpoint = utils::get_alternative_reddit_url_base();
+		let url = format!("{auth_endpoint}/api/v1/access_token");
+		let mut builder = CLIENT.post(&url);
 
 		// Add minimal headers
-		builder = builder.header("Host", "www.reddit.com");
+		builder = builder.header("Host", auth_endpoint);
 		builder = builder.header("User-Agent", &self.user_agent);
 		builder = builder.header("Accept", "*/*");
 		builder = builder.header("Accept-Language", "en-US,en;q=0.5");
@@ -402,7 +401,7 @@ impl OauthBackend for GenericWebAuth {
 		);
 
 		// Insert a few necessary headers
-		self.additional_headers.insert("Origin".to_owned(), "https://www.reddit.com".to_owned());
+		self.additional_headers.insert("Origin".to_owned(), auth_endpoint.to_owned());
 		self.additional_headers.insert("User-Agent".to_owned(), self.user_agent.to_owned());
 
 		Ok(OauthResponse {