first commit
This commit is contained in:
Elaina Claus
271
src/filecache/mod.rs
Executable file
271
src/filecache/mod.rs
Executable file
@@ -0,0 +1,271 @@
|
||||
#![allow(dead_code)]
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::fs::File;
|
||||
use std::io::prelude::*;
|
||||
use std::io::{Cursor, ErrorKind};
|
||||
use std::sync::{
|
||||
mpsc::{channel, Receiver},
|
||||
Arc, RwLock,
|
||||
};
|
||||
|
||||
use std::thread;
|
||||
use std::time::Duration;
|
||||
|
||||
use notify::{DebouncedEvent, RecommendedWatcher, RecursiveMode, Watcher};
|
||||
|
||||
type CacheGuard<T> = Arc<RwLock<T>>;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum FileEntryError {
|
||||
NeedsUpdate,
|
||||
EmptyFile,
|
||||
FileOpenError,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct FileEntry {
|
||||
path: CacheGuard<std::path::PathBuf>,
|
||||
hash: CacheGuard<Option<blake3::Hash>>,
|
||||
_content: CacheGuard<Option<std::io::BufReader<File>>>,
|
||||
}
|
||||
|
||||
impl FileEntry {
|
||||
pub fn new(file_path: &std::path::Path) -> FileEntry {
|
||||
let mut fe = FileEntry {
|
||||
path: Arc::new(RwLock::new(file_path.to_path_buf())),
|
||||
hash: Arc::new(RwLock::new(None)),
|
||||
_content: Arc::new(RwLock::new(None)),
|
||||
};
|
||||
|
||||
fe.do_hash();
|
||||
log::trace!("{:#?}", fe);
|
||||
|
||||
fe
|
||||
}
|
||||
|
||||
pub fn get_hash_string(&self) -> String {
|
||||
let hash_ptr = self.hash.read().unwrap();
|
||||
hash_ptr.unwrap().to_string()
|
||||
}
|
||||
|
||||
pub fn open_path(&self) -> CacheGuard<String> {
|
||||
let ptr = self.path.clone();
|
||||
let path = ptr.read().unwrap();
|
||||
let path = path.as_path().to_string_lossy();
|
||||
|
||||
Arc::new(RwLock::new(String::from(&*path)))
|
||||
}
|
||||
|
||||
// HACK: this whole function has been hacked up trying to fix dead locks.
|
||||
fn do_hash(&mut self) {
|
||||
let path_ptr = self.path.clone();
|
||||
let path_lock = path_ptr.read().unwrap();
|
||||
|
||||
if path_lock.is_file() {
|
||||
let path_lock = self.path.clone();
|
||||
let file = std::fs::File::open(path_lock.read().unwrap().as_path());
|
||||
let file = file.unwrap_or_else(|_| panic!("issue opening file for {:#?}", &self.path));
|
||||
|
||||
let temp_content = unsafe {
|
||||
match memmap2::Mmap::map(&file) {
|
||||
Ok(mm) => Cursor::new(mm),
|
||||
Err(_) => panic!(),
|
||||
}
|
||||
};
|
||||
|
||||
let mut hasher = blake3::Hasher::new();
|
||||
// HACK: this feels bad...
|
||||
hasher.update_rayon(
|
||||
&temp_content
|
||||
.bytes()
|
||||
// FIXME: clippy suggestion below
|
||||
// warning: `filter(..).map(..)` can be simplified as `filter_map(..)`
|
||||
// --> src/filecache/mod.rs:82:22
|
||||
// |
|
||||
//82 | .filter(|b| b.is_ok())
|
||||
// | ______________________^
|
||||
//83 | | .map(|b| b.unwrap())
|
||||
// | |________________________________________^ help: try: `filter_map(|b| b.ok())`
|
||||
// |
|
||||
// = note: `#[warn(clippy::manual_filter_map)]` on by default
|
||||
// = help: for further information visit https://rust-lang.github.io/rust-clippy/master/index.html#manual_filter_map
|
||||
|
||||
.filter(|b| b.is_ok())
|
||||
.map(|b| b.unwrap())
|
||||
.collect::<Vec<_>>(),
|
||||
);
|
||||
|
||||
*self.hash.write().unwrap() = Some(hasher.finalize());
|
||||
}
|
||||
/*
|
||||
|
||||
loop {
|
||||
let has_content = ptr.read().unwrap().is_some();
|
||||
if has_content {
|
||||
let mut hasher = blake3::Hasher::new();
|
||||
hasher.update_rayon(content.get_ref());
|
||||
|
||||
|
||||
let hash_ptr = self.hash.clone();
|
||||
let hash_ptr = hash_ptr.write().unwrap();
|
||||
let hash_ptr = Some(hasher.finalize());
|
||||
|
||||
break;
|
||||
} else {
|
||||
|
||||
let content_lock = self.content.clone();
|
||||
let mut content = content_lock.write().unwrap();
|
||||
|
||||
if content.is_none() {
|
||||
let path_lock = self.path.clone();
|
||||
let file = std::fs::File::open(path_lock.read().unwrap().as_path());
|
||||
let file = file.unwrap_or_else(|_| panic!("issue opening file for {:#?}", &self.path));
|
||||
|
||||
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
*/
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum CacheEntryError {
|
||||
NotFound,
|
||||
FileLocked,
|
||||
FileExists,
|
||||
}
|
||||
|
||||
pub struct FileCache {
|
||||
cache: CacheGuard<HashMap<String, CacheGuard<FileEntry>>>,
|
||||
cache_dir: std::path::PathBuf,
|
||||
notify_watcher: RecommendedWatcher,
|
||||
notify_thread: std::thread::JoinHandle<()>,
|
||||
}
|
||||
|
||||
impl FileCache {
|
||||
pub fn new(cache_dir: &std::path::Path) -> FileCache {
|
||||
let (tx, rx) = channel();
|
||||
|
||||
match std::fs::create_dir(&cache_dir) {
|
||||
Ok(_) => log::info!(
|
||||
"Created new file directory @ {}",
|
||||
cache_dir.to_string_lossy()
|
||||
),
|
||||
Err(e) => match e.kind() {
|
||||
ErrorKind::AlreadyExists => {
|
||||
log::info!("Attempted to create a new dir: {} for FileCache, but it already exists (This is normal)", cache_dir.to_string_lossy());
|
||||
}
|
||||
ErrorKind::PermissionDenied => {
|
||||
log::error!("Attempted to create a new dir: {} for FileCache, but permission was denined\n{}", cache_dir.to_string_lossy(), &e)
|
||||
}
|
||||
_ => {
|
||||
log::error!("{}", &e)
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
let mut fc = FileCache {
|
||||
cache: Arc::new(RwLock::new(HashMap::new())),
|
||||
cache_dir: cache_dir.to_path_buf(),
|
||||
notify_watcher: notify::Watcher::new(tx, Duration::from_secs(1)).unwrap(),
|
||||
notify_thread: thread::Builder::new()
|
||||
.name("notify-thread".to_string())
|
||||
.spawn(move || FileCache::notify_loop(rx))
|
||||
.unwrap(),
|
||||
};
|
||||
fc.notify_watcher
|
||||
.watch(&fc.cache_dir, RecursiveMode::Recursive)
|
||||
.unwrap();
|
||||
|
||||
fc
|
||||
}
|
||||
|
||||
pub fn get(&self, file_hash: String) -> Result<CacheGuard<FileEntry>, CacheEntryError> {
|
||||
let cache_lock = Arc::clone(&self.cache);
|
||||
let cache_lock = cache_lock.read().unwrap();
|
||||
|
||||
// FIXME: clippy suggestion below
|
||||
// warning: temporary with significant `Drop` in `match` scrutinee will live until the end of the `match` expression
|
||||
// --> src/filecache/mod.rs:178:15
|
||||
// |
|
||||
// 178 | match cache_lock.get(&file_hash) {
|
||||
// | ^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
// ...
|
||||
// 181 | }
|
||||
// | - temporary lives until here
|
||||
// |
|
||||
// = note: `#[warn(clippy::significant_drop_in_scrutinee)]` on by default
|
||||
// = note: this might lead to deadlocks or other unexpected behavior
|
||||
// = help: for further information visit https://rust-lang.github.io/rust-clippy/master/index.html#significant_drop_in_scrutinee
|
||||
|
||||
match cache_lock.get(&file_hash) {
|
||||
Some(fe) => Ok(fe.clone()),
|
||||
None => Err(CacheEntryError::NotFound),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn add(&self, file_path: &std::path::Path) -> Result<String, CacheEntryError> {
|
||||
let fe = FileEntry::new(file_path);
|
||||
let hash = fe.get_hash_string();
|
||||
|
||||
let cache_lock = self.cache.clone();
|
||||
let mut cache_lock = cache_lock.write().unwrap();
|
||||
|
||||
cache_lock
|
||||
.entry(fe.get_hash_string())
|
||||
.or_insert_with(|| Arc::new(RwLock::new(fe)));
|
||||
|
||||
Ok(hash)
|
||||
}
|
||||
|
||||
fn notify_loop(rx: Receiver<DebouncedEvent>) {
|
||||
let this_thread = std::thread::current();
|
||||
log::info!("notify loop starting on thread-{:?}", &this_thread);
|
||||
|
||||
let thread_name: String = match &this_thread.name() {
|
||||
Some(s) => s.to_string(),
|
||||
None => "notify-thread".to_string(),
|
||||
};
|
||||
|
||||
loop {
|
||||
match rx.recv() {
|
||||
Ok(event) => {
|
||||
log::info!("[{}] {:?}", thread_name, &event);
|
||||
|
||||
// FIXME: need to cover all event types with appropiate actions and ignore the rest.
|
||||
match event {
|
||||
DebouncedEvent::NoticeWrite(_) => {
|
||||
// do nothing, this is sent when a file is being updated
|
||||
}
|
||||
DebouncedEvent::NoticeRemove(_) => {
|
||||
// do nothing, this is sent when a file is being removed
|
||||
}
|
||||
DebouncedEvent::Create(p) => {
|
||||
match crate::FILECACHE.add(p.as_path()) {
|
||||
Ok(hash) => log::info!("[{}] Found new file @ {} ({})",thread_name, &p.as_path().to_string_lossy(), hash.to_string()),
|
||||
Err(e) => log::error!("[{}] Found a new file but there was an error adding to internal cache\nFile -> {} \nError -> {:#?}", thread_name, &p.as_path().to_string_lossy(), &e),
|
||||
}
|
||||
}
|
||||
DebouncedEvent::Write(p) => {
|
||||
match crate::FILECACHE.add(p.as_path()) {
|
||||
Ok(hash) => log::info!("[{}] A file was updated @ {} ({})",thread_name, &p.as_path().to_string_lossy(), hash.to_string()),
|
||||
Err(e) => log::error!("[{}] Found a new file but there was an error updating the internal cache\nFile -> {} \nError -> {:#?}", thread_name, &p.as_path().to_string_lossy(), &e),
|
||||
}
|
||||
}
|
||||
DebouncedEvent::Chmod(_) => {},
|
||||
DebouncedEvent::Remove(_) => {}
|
||||
DebouncedEvent::Rename(_, _) => {}
|
||||
DebouncedEvent::Rescan => {},
|
||||
DebouncedEvent::Error(_, _) => log::debug!(
|
||||
"received a Error event on watched dir, but we don't do anything!"
|
||||
),
|
||||
}
|
||||
}
|
||||
Err(e) => log::info!("watch error: {:?}", e),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
13
src/html/upload.html
Executable file
13
src/html/upload.html
Executable file
@@ -0,0 +1,13 @@
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<body>
|
||||
|
||||
<h1>rfile File upload</h1>
|
||||
|
||||
<form action="/" method="POST" enctype=multipart/form-data target="_self">
|
||||
<input type="file" id="myFile" name="data">
|
||||
<input type="submit" value="Submit">
|
||||
</form>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
139
src/main.rs
Executable file
139
src/main.rs
Executable file
@@ -0,0 +1,139 @@
|
||||
#![feature(proc_macro_hygiene, decl_macro)]
|
||||
#![feature(seek_stream_len)]
|
||||
#![feature(let_else)]
|
||||
|
||||
#[allow(unused_imports)]
|
||||
#[allow(unused_macros)]
|
||||
/* Logging crates */
|
||||
#[macro_use]
|
||||
extern crate log;
|
||||
|
||||
/* Parallelism and multithreading crates */
|
||||
extern crate futures;
|
||||
extern crate rayon;
|
||||
extern crate tokio;
|
||||
|
||||
/* random number generation */
|
||||
extern crate rand;
|
||||
|
||||
/* http support crates */
|
||||
#[macro_use]
|
||||
extern crate rocket;
|
||||
|
||||
extern crate mime;
|
||||
|
||||
/* compression crates */
|
||||
//extern crate flate2;
|
||||
//extern crate tar;
|
||||
|
||||
/* crypt crates */
|
||||
extern crate data_encoding;
|
||||
//extern crate ring;
|
||||
|
||||
/* database crates */
|
||||
//extern crate rusqlite;
|
||||
|
||||
/* filesystem utility crates */
|
||||
extern crate notify;
|
||||
|
||||
/* general utility crates */
|
||||
//extern crate chrono;
|
||||
extern crate lazy_static;
|
||||
extern crate regex;
|
||||
|
||||
mod filecache;
|
||||
mod routes;
|
||||
|
||||
use std::{fs, path::PathBuf};
|
||||
|
||||
use filecache::*;
|
||||
use lazy_static::lazy_static;
|
||||
use rocket::fairing::AdHoc;
|
||||
|
||||
static CACHE_DIR: &str = "CACHE/";
|
||||
|
||||
lazy_static! {
|
||||
static ref FILECACHE: FileCache = {
|
||||
let mut cwd = std::env::current_dir().unwrap();
|
||||
cwd.push(CACHE_DIR);
|
||||
|
||||
FileCache::new(&cwd)
|
||||
};
|
||||
}
|
||||
|
||||
lazy_static! {
|
||||
static ref CACHE_PATH: PathBuf = {
|
||||
let mut cwd = std::env::current_dir().unwrap();
|
||||
cwd.push(&CACHE_DIR);
|
||||
|
||||
cwd
|
||||
};
|
||||
}
|
||||
|
||||
#[rocket::main]
|
||||
async fn main() -> Result<(), rocket::Error> {
|
||||
let _rocket = rocket::build()
|
||||
.mount(
|
||||
"/",
|
||||
rocket::routes![
|
||||
routes::index,
|
||||
routes::download_file,
|
||||
routes::query_file,
|
||||
routes::upload_file
|
||||
],
|
||||
)
|
||||
.attach(AdHoc::on_liftoff("Application setup", |_| {
|
||||
Box::pin(async move {
|
||||
lazy_static::initialize(&CACHE_PATH);
|
||||
lazy_static::initialize(&FILECACHE);
|
||||
|
||||
let keep_file = {
|
||||
let mut marker_path = PathBuf::from(&CACHE_PATH.as_path());
|
||||
marker_path.push("rfile.meta");
|
||||
|
||||
marker_path
|
||||
};
|
||||
|
||||
if !(&CACHE_PATH.is_dir()) {
|
||||
match fs::create_dir(&CACHE_PATH.as_path()) {
|
||||
Ok(_) => log::info!(
|
||||
"crated new cache directory @ {}",
|
||||
&CACHE_PATH.as_path().to_string_lossy()
|
||||
),
|
||||
Err(e) => {
|
||||
log::error!("Error creating cache directory - {}", &e);
|
||||
panic!("{}", &e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !keep_file.exists() {
|
||||
match fs::File::create(&keep_file) {
|
||||
Ok(_) => log::info!(
|
||||
"crated new enviroment file @ {}",
|
||||
&CACHE_PATH.as_path().to_string_lossy()
|
||||
),
|
||||
Err(e) => {
|
||||
log::error!("Error creating enviroment file - {}", &e);
|
||||
panic!("{}", &e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
match FILECACHE.add(&keep_file) {
|
||||
Ok(_) => {
|
||||
log::info!("added {} to cache", &keep_file.as_path().to_string_lossy())
|
||||
}
|
||||
Err(e) => match e {
|
||||
CacheEntryError::NotFound => todo!(),
|
||||
CacheEntryError::FileLocked => todo!(),
|
||||
CacheEntryError::FileExists => todo!(),
|
||||
},
|
||||
}
|
||||
})
|
||||
}))
|
||||
.ignite().await?
|
||||
.launch().await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
161
src/routes/mod.rs
Executable file
161
src/routes/mod.rs
Executable file
@@ -0,0 +1,161 @@
|
||||
use std::path::PathBuf;
|
||||
|
||||
use rocket::{Response, Request, response};
|
||||
use rocket::form::Form;
|
||||
use rocket::fs::{NamedFile, TempFile};
|
||||
use rocket::http::{ContentType, Status};
|
||||
use rocket::response::{content, Debug};
|
||||
use rocket::response::Responder;
|
||||
|
||||
static UPLOAD_HTML: &str = include_str!("../html/upload.html");
|
||||
|
||||
#[get("/")]
|
||||
pub async fn index() -> Result<content::RawHtml<&'static str>, Debug<std::io::Error>> {
|
||||
let response = content::RawHtml(UPLOAD_HTML);
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
// TODO: This should probably be moved to FileCache and a responder made for FileEntries
|
||||
// currently we get a ref to a RwLock<FileEntry> that we lock for read access and grab the path
|
||||
// to pass to a new NamedFile which is then wrapped in a CachedFile<T>
|
||||
pub struct CachedFile(NamedFile);
|
||||
|
||||
// Custom responder for the wrapper CachedFile, uses the response the NamedFile would return
|
||||
// and sets Content-Disposition(file type and filename) and Cache-control
|
||||
impl<'r> Responder<'r, 'static> for CachedFile {
|
||||
fn respond_to(self, req: &'r Request) -> response::Result<'static> {
|
||||
|
||||
let name = match self.0.path().file_name() {
|
||||
Some(f) => {
|
||||
f.to_string_lossy().to_string()
|
||||
},
|
||||
None => todo!()
|
||||
};
|
||||
|
||||
Response::build_from(self.0.respond_to(req)?)
|
||||
.raw_header("Content-Disposition", format!("application/octet-stream; filename=\"{}\"", name))
|
||||
.raw_header("Cache-control", "max-age=86400") // 24h (24*60*60)
|
||||
.ok()
|
||||
}
|
||||
}
|
||||
|
||||
#[get("/download/<file_hash>")]
|
||||
pub async fn download_file(file_hash: String) -> Option<CachedFile> {
|
||||
let file = match crate::FILECACHE.get(file_hash) {
|
||||
Ok(fe) => fe,
|
||||
Err(_) => {
|
||||
panic!()
|
||||
}
|
||||
};
|
||||
|
||||
let file = {
|
||||
let path_lock = file.read().unwrap().open_path();
|
||||
let path = path_lock.read().unwrap().clone();
|
||||
|
||||
match NamedFile::open(path).await.ok() {
|
||||
Some(f) => f,
|
||||
None => todo!()
|
||||
}
|
||||
};
|
||||
|
||||
log::debug!("{:#?}", NamedFile::path(&file));
|
||||
Some(CachedFile(file))
|
||||
}
|
||||
|
||||
#[get("/search/<query>")]
|
||||
pub async fn query_file(query: String) -> String {
|
||||
// TODO: implement some sort of search function...maybe...
|
||||
format!("[NOT IMPLEMENTED] Looking up {}...", &query)
|
||||
}
|
||||
|
||||
// FIXME: Clippy suggestion (Issue is in rocket's implementation of #[derive(FromForm)] )
|
||||
// warning: unnecessary closure used with `bool::then`
|
||||
// --> src/routes/mod.rs:71:11
|
||||
// |
|
||||
// 71 | data: TempFile<'f>,
|
||||
// | ^^^^^^^^^^^^ help: use `then_some(..)` instead: `then_some(data)`
|
||||
// |
|
||||
// = note: `#[warn(clippy::unnecessary_lazy_evaluations)]` on by default
|
||||
// = help: for further information visit https://rust-lang.github.io/rust-clippy/master/index.html#unnecessary_lazy_evaluations
|
||||
|
||||
#[derive(FromForm)]
|
||||
pub struct UploadFile<'f> {
|
||||
data: TempFile<'f>,
|
||||
}
|
||||
|
||||
#[post("/", data = "<form>")]
|
||||
pub async fn upload_file(mut form: Form<UploadFile<'_>>) -> Result<String, String> {
|
||||
//this is how a POST looks from our web form (simplified)
|
||||
//Content-Disposition: form-data; name="filename"; filename="1510287646273.png"
|
||||
//Content-Type: image/png
|
||||
// <left blank>
|
||||
//---DATA---
|
||||
|
||||
let filename = match form.data.name() {
|
||||
Some(s) => String::from(s),
|
||||
None => return Err(Status::BadRequest.to_string()),
|
||||
};
|
||||
|
||||
let raw_filename = match form.data.raw_name() {
|
||||
Some(s) => <&rocket::fs::FileName>::clone(&s),
|
||||
None => return Err(Status::BadRequest.to_string()),
|
||||
};
|
||||
|
||||
let filetype = match form.data.content_type() {
|
||||
Some(ct) => ct.clone(),
|
||||
None => ContentType::Plain,
|
||||
};
|
||||
|
||||
let extension = match filetype.extension() {
|
||||
Some(ext) => ext.to_string(),
|
||||
None => {
|
||||
let mut name = String::new();
|
||||
let unsafe_name = raw_filename.dangerous_unsafe_unsanitized_raw();
|
||||
|
||||
if unsafe_name.len() < 255 {
|
||||
name = match unsafe_name.as_str().trim().split_once('.') {
|
||||
Some((_, ext)) => {
|
||||
let mut ext = ext.replace("..", "").replace('/', "").replace(':', "");
|
||||
|
||||
if ext.starts_with('.') {
|
||||
ext = ext[1..].to_string();
|
||||
}
|
||||
|
||||
ext
|
||||
}
|
||||
None => "badext".to_string(),
|
||||
};
|
||||
}
|
||||
log::debug!(
|
||||
"unhandled extension uploaded...parse attempt ended up with {}",
|
||||
&name
|
||||
);
|
||||
|
||||
name
|
||||
}
|
||||
};
|
||||
|
||||
let filepath = PathBuf::from(format!(
|
||||
"{}{}.{}",
|
||||
&crate::CACHE_PATH.as_path().display(),
|
||||
&filename,
|
||||
extension
|
||||
));
|
||||
let copy_res = match form.data.copy_to(filepath.as_path()).await {
|
||||
Ok(_) => Ok(()),
|
||||
Err(e) => Err(e.to_string()),
|
||||
};
|
||||
|
||||
let hasher = crate::FileEntry::new(filepath.as_path()).get_hash_string();
|
||||
|
||||
match (copy_res, hasher) {
|
||||
(Ok(_), h) => Ok(format!(
|
||||
"OK...received {}.{} size:{} bytes\nBLAKE3 Hash: {}",
|
||||
&filename,
|
||||
&extension,
|
||||
form.data.len(),
|
||||
&h
|
||||
)),
|
||||
(Err(_), _) => Err(Status::InternalServerError.to_string()),
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user