Browse Source

Configurable per-website blacklist time, better rng

master
Stephen 4 months ago
parent
commit
db3c7cc5aa
4 changed files with 38 additions and 21 deletions
  1. +1
    -1
      Cargo.lock
  2. +2
    -2
      Cargo.toml
  3. +3
    -1
      config.example.toml
  4. +32
    -17
      src/main.rs

+ 1
- 1
Cargo.lock View File

@ -704,7 +704,7 @@ dependencies = [
[[package]]
name = "proxy_manager"
version = "0.2.0"
version = "0.3.0"
dependencies = [
"chrono",
"rand 0.7.3",


+ 2
- 2
Cargo.toml View File

@ -1,6 +1,6 @@
[package]
name = "proxy_manager"
version = "0.2.0"
version = "0.3.0"
authors = ["Stephen D <@stephen:m.scd31.com>"]
edition = "2018"
description = "Manages web proxies across different projects/processes"
@ -13,7 +13,7 @@ serde = { version = "1.0", features = ["derive"] }
chrono = "0.4.15"
warp = "0.2"
tokio = { version = "0.2", features = ["macros"] }
rand = "0.7.3"
rand = "0.7"
[package.metadata.deb]
maintainer = "Stephen D <@stephen:m.scd31.com>"


+ 3
- 1
config.example.toml View File

@ -8,6 +8,8 @@ proxies = [
[websites.facebook]
delay = 5 # number of seconds between requests, per proxy
blacklist_minutes = 15 # how many minutes to wait if the proxy gets blacklisted, before we allow it to be served again
[websites.google]
delay = 0 # no delay
delay = 0 # no delay
blacklist_minutes = 0

+ 32
- 17
src/main.rs View File

@ -1,20 +1,22 @@
use rand::prelude::StdRng;
use rand::seq::SliceRandom;
use rand::thread_rng;
use rand::{thread_rng, SeedableRng};
use serde::Deserialize;
use std::collections::HashMap;
use std::ops::DerefMut;
use std::sync::{Arc, Mutex, RwLock};
use std::time::{Duration, Instant};
use std::{cmp, fs};
use tokio::time;
use warp::{http, Filter, Reply};
const BLACKLIST_DELAY_TIME_MINS: u64 = 15;
const ONE_YEAR: Duration = Duration::from_secs(60 * 60 * 24 * 365);
#[derive(Deserialize, Debug)]
struct WebsiteConfig {
// May add more here in the future
delay: u32, // seconds
blacklist_minutes: u32,
}
#[derive(Deserialize, Debug)]
@ -28,12 +30,14 @@ struct Website {
proxies: HashMap<String, Instant>,
index: usize,
delay: u32,
blacklist_minutes: u32,
}
#[derive(Clone)]
struct ServerState {
websites: HashMap<String, Arc<Mutex<Website>>>,
proxies: Vec<String>,
rng: Arc<Mutex<StdRng>>,
}
#[derive(Deserialize)]
@ -76,6 +80,7 @@ async fn main() {
proxies: hsh,
index: 0,
delay: config.websites[site].delay,
blacklist_minutes: config.websites[site].blacklist_minutes,
})),
);
}
@ -83,6 +88,7 @@ async fn main() {
let state = Arc::new(RwLock::new(ServerState {
websites,
proxies: config.proxies,
rng: Arc::new(Mutex::new(StdRng::from_rng(thread_rng()).unwrap())),
}));
let get_proxy = {
@ -120,7 +126,11 @@ async fn get_proxy_fn(
.iter()
.map(|(k, v)| (k.clone(), *v))
.collect();
proxies.shuffle(&mut thread_rng());
//for _ in 0..100 {
proxies.shuffle(state.rng.lock().unwrap().deref_mut());
//}
println!("{:?}", proxies);
let mut min_proxy_address: String = "".to_string();
let mut min_time = Instant::now() + ONE_YEAR;
@ -158,7 +168,7 @@ async fn get_proxy_fn(
}
// Marks a proxy as blacklisted(not working, blocked by website, etc.)
// Will auto-unblacklist after 15 minutes
// Will auto-un-blacklist after 15 minutes
async fn blacklist_proxy_fn(
website_str: String,
state: Arc<RwLock<ServerState>>,
@ -166,21 +176,26 @@ async fn blacklist_proxy_fn(
) -> Result<impl warp::Reply, warp::Rejection> {
let state = state.read().unwrap();
match state.websites.get(&website_str) {
Some(x) => match x.lock().unwrap().proxies.get_mut(&body.proxy) {
Some(y) => {
*y = Instant::now() + Duration::from_secs(60 * BLACKLIST_DELAY_TIME_MINS);
Ok(
warp::reply::with_status("Success", http::status::StatusCode::OK)
.into_response(),
Some(x) => {
let mut website = x.lock().unwrap();
let blacklist_minutes = website.blacklist_minutes as u64;
match website.proxies.get_mut(&body.proxy) {
Some(y) => {
*y = Instant::now() + Duration::from_secs(60 * blacklist_minutes);
Ok(
warp::reply::with_status("Success", http::status::StatusCode::OK)
.into_response(),
)
}
None => Ok(warp::reply::with_status(
"Proxy not found",
http::status::StatusCode::NOT_FOUND,
)
.into_response()),
}
None => Ok(warp::reply::with_status(
"Proxy not found",
http::status::StatusCode::NOT_FOUND,
)
.into_response()),
},
}
None => Ok(warp::reply::with_status(
"Website not found",
http::status::StatusCode::NOT_FOUND,


Loading…
Cancel
Save