nivirx/rfile
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

first commit

Browse Source
This commit is contained in:
Elaina Claus
2023-02-28 22:17:21 -05:00
commit 42e43c107c
11 changed files with 2874 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored Executable file
View File

@@ -0,0 +1,3 @@
/target
/files
.DS_Store

45
.vscode/launch.json vendored Executable file
View File

@@ -0,0 +1,45 @@
{
// Use IntelliSense to learn about possible attributes.
// Hover to view descriptions of existing attributes.
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"configurations": [
{
"type": "lldb",
"request": "launch",
"name": "Debug executable 'rfile'",
"cargo": {
"args": [
"build",
"--bin=rfile",
"--package=rfile"
],
"filter": {
"name": "rfile",
"kind": "bin"
}
},
"args": [],
"cwd": "${workspaceFolder}/target/debug/"
},
{
"type": "lldb",
"request": "launch",
"name": "Debug unit tests in executable 'rfile'",
"cargo": {
"args": [
"test",
"--no-run",
"--bin=rfile",
"--package=rfile"
],
"filter": {
"name": "rfile",
"kind": "bin"
}
},
"args": [],
"cwd": "${workspaceFolder}"
}
]
}

0
.vscode/settings.json vendored Executable file
View File

2182
Cargo.lock generated Executable file
View File

File diff suppressed because it is too large Load Diff

42
Cargo.toml Executable file
View File

@@ -0,0 +1,42 @@
[package]
name = "rfile"
version = "0.2.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
# basic logging utilities
# Rocket sets up its own logger so we just need the log crate to talk to its global hook
log = "0.4.17"
# some libs for easy async and threading
futures = "0.3.23"
tokio = { version = "1.20.1", features = ["full"] }
rayon = "1.5.3"
# file change events
notify = "4.0.17"
#http related stuff
rocket = "0.5.0-rc.2"
mime = "0.3.16"
# database
rusqlite = "0.28.0"
# crypt stuff
#ring = { version = "0.16.20", features = ["std"] }
data-encoding = "2.3.2"
blake3 = { version = "1.3.1", features = ["std", "rayon"] }
# compression
flate2 = { version = "1.0.24" }
tar = "0.4.38"
# and some stuff I don't know why it isn't in std
rand = { version = "0.8.5", features = ["std", "alloc", "getrandom", "std_rng", "log", "nightly", "simd_support"] }
lazy_static = "1.4.0"
regex = "1.5.6"
chrono = "0.4.22"
memmap2 = "0.5.7"

1
README.md Executable file
View File

@@ -0,0 +1 @@
Web Application for file sharing written in Rust+Rocket

17
Rocket.toml Executable file
View File

@@ -0,0 +1,17 @@
[debug]
address = "127.0.0.1"
port = 8080
workers = 4
keep_alive = 5
log_level = "debug"
ident = "rfile"
limits = { bytes = "8KiB", data-form = "256MiB", file = "256MiB", form = "64KiB", json = "1MiB", msgpack = "1MiB", string = "8KiB" }
[release]
address = "127.0.0.1"
port = 80
workers = 8
keep_alive = 5
log_level = "normal"
ident = "rfile"
limits = { bytes = "8KiB", data-form = "256MiB", file = "256MiB", form = "64KiB", json = "1MiB", msgpack = "1MiB", string = "8KiB" }

271
src/filecache/mod.rs Executable file
View File

@@ -0,0 +1,271 @@
#![allow(dead_code)]
use std::collections::HashMap;
use std::fs::File;
use std::io::prelude::*;
use std::io::{Cursor, ErrorKind};
use std::sync::{
mpsc::{channel, Receiver},
Arc, RwLock,
};
use std::thread;
use std::time::Duration;
use notify::{DebouncedEvent, RecommendedWatcher, RecursiveMode, Watcher};
type CacheGuard<T> = Arc<RwLock<T>>;
#[derive(Debug)]
pub enum FileEntryError {
NeedsUpdate,
EmptyFile,
FileOpenError,
}
#[derive(Debug)]
pub struct FileEntry {
path: CacheGuard<std::path::PathBuf>,
hash: CacheGuard<Option<blake3::Hash>>,
_content: CacheGuard<Option<std::io::BufReader<File>>>,
}
impl FileEntry {
pub fn new(file_path: &std::path::Path) -> FileEntry {
let mut fe = FileEntry {
path: Arc::new(RwLock::new(file_path.to_path_buf())),
hash: Arc::new(RwLock::new(None)),
_content: Arc::new(RwLock::new(None)),
};
fe.do_hash();
log::trace!("{:#?}", fe);
fe
}
pub fn get_hash_string(&self) -> String {
let hash_ptr = self.hash.read().unwrap();
hash_ptr.unwrap().to_string()
}
pub fn open_path(&self) -> CacheGuard<String> {
let ptr = self.path.clone();
let path = ptr.read().unwrap();
let path = path.as_path().to_string_lossy();
Arc::new(RwLock::new(String::from(&*path)))
}
// HACK: this whole function has been hacked up trying to fix dead locks.
fn do_hash(&mut self) {
let path_ptr = self.path.clone();
let path_lock = path_ptr.read().unwrap();
if path_lock.is_file() {
let path_lock = self.path.clone();
let file = std::fs::File::open(path_lock.read().unwrap().as_path());
let file = file.unwrap_or_else(|_| panic!("issue opening file for {:#?}", &self.path));
let temp_content = unsafe {
match memmap2::Mmap::map(&file) {
Ok(mm) => Cursor::new(mm),
Err(_) => panic!(),
}
};
let mut hasher = blake3::Hasher::new();
// HACK: this feels bad...
hasher.update_rayon(
&temp_content
.bytes()
// FIXME: clippy suggestion below
// warning: `filter(..).map(..)` can be simplified as `filter_map(..)`
// --> src/filecache/mod.rs:82:22
// |
//82 | .filter(|b| b.is_ok())
// | ______________________^
//83 | | .map(|b| b.unwrap())
// | |________________________________________^ help: try: `filter_map(|b| b.ok())`
// |
// = note: `#[warn(clippy::manual_filter_map)]` on by default
// = help: for further information visit https://rust-lang.github.io/rust-clippy/master/index.html#manual_filter_map
.filter(|b| b.is_ok())
.map(|b| b.unwrap())
.collect::<Vec<_>>(),
);
*self.hash.write().unwrap() = Some(hasher.finalize());
}
/*
loop {
let has_content = ptr.read().unwrap().is_some();
if has_content {
let mut hasher = blake3::Hasher::new();
hasher.update_rayon(content.get_ref());
let hash_ptr = self.hash.clone();
let hash_ptr = hash_ptr.write().unwrap();
let hash_ptr = Some(hasher.finalize());
break;
} else {
let content_lock = self.content.clone();
let mut content = content_lock.write().unwrap();
if content.is_none() {
let path_lock = self.path.clone();
let file = std::fs::File::open(path_lock.read().unwrap().as_path());
let file = file.unwrap_or_else(|_| panic!("issue opening file for {:#?}", &self.path));
continue;
}
}
*/
}
}
#[derive(Debug)]
pub enum CacheEntryError {
NotFound,
FileLocked,
FileExists,
}
pub struct FileCache {
cache: CacheGuard<HashMap<String, CacheGuard<FileEntry>>>,
cache_dir: std::path::PathBuf,
notify_watcher: RecommendedWatcher,
notify_thread: std::thread::JoinHandle<()>,
}
impl FileCache {
pub fn new(cache_dir: &std::path::Path) -> FileCache {
let (tx, rx) = channel();
match std::fs::create_dir(&cache_dir) {
Ok(_) => log::info!(
"Created new file directory @ {}",
cache_dir.to_string_lossy()
),
Err(e) => match e.kind() {
ErrorKind::AlreadyExists => {
log::info!("Attempted to create a new dir: {} for FileCache, but it already exists (This is normal)", cache_dir.to_string_lossy());
}
ErrorKind::PermissionDenied => {
log::error!("Attempted to create a new dir: {} for FileCache, but permission was denined\n{}", cache_dir.to_string_lossy(), &e)
}
_ => {
log::error!("{}", &e)
}
},
}
let mut fc = FileCache {
cache: Arc::new(RwLock::new(HashMap::new())),
cache_dir: cache_dir.to_path_buf(),
notify_watcher: notify::Watcher::new(tx, Duration::from_secs(1)).unwrap(),
notify_thread: thread::Builder::new()
.name("notify-thread".to_string())
.spawn(move || FileCache::notify_loop(rx))
.unwrap(),
};
fc.notify_watcher
.watch(&fc.cache_dir, RecursiveMode::Recursive)
.unwrap();
fc
}
pub fn get(&self, file_hash: String) -> Result<CacheGuard<FileEntry>, CacheEntryError> {
let cache_lock = Arc::clone(&self.cache);
let cache_lock = cache_lock.read().unwrap();
// FIXME: clippy suggestion below
// warning: temporary with significant `Drop` in `match` scrutinee will live until the end of the `match` expression
// --> src/filecache/mod.rs:178:15
// |
// 178 | match cache_lock.get(&file_hash) {
// | ^^^^^^^^^^^^^^^^^^^^^^^^^^
// ...
// 181 | }
// | - temporary lives until here
// |
// = note: `#[warn(clippy::significant_drop_in_scrutinee)]` on by default
// = note: this might lead to deadlocks or other unexpected behavior
// = help: for further information visit https://rust-lang.github.io/rust-clippy/master/index.html#significant_drop_in_scrutinee
match cache_lock.get(&file_hash) {
Some(fe) => Ok(fe.clone()),
None => Err(CacheEntryError::NotFound),
}
}
pub fn add(&self, file_path: &std::path::Path) -> Result<String, CacheEntryError> {
let fe = FileEntry::new(file_path);
let hash = fe.get_hash_string();
let cache_lock = self.cache.clone();
let mut cache_lock = cache_lock.write().unwrap();
cache_lock
.entry(fe.get_hash_string())
.or_insert_with(|| Arc::new(RwLock::new(fe)));
Ok(hash)
}
fn notify_loop(rx: Receiver<DebouncedEvent>) {
let this_thread = std::thread::current();
log::info!("notify loop starting on thread-{:?}", &this_thread);
let thread_name: String = match &this_thread.name() {
Some(s) => s.to_string(),
None => "notify-thread".to_string(),
};
loop {
match rx.recv() {
Ok(event) => {
log::info!("[{}] {:?}", thread_name, &event);
// FIXME: need to cover all event types with appropiate actions and ignore the rest.
match event {
DebouncedEvent::NoticeWrite(_) => {
// do nothing, this is sent when a file is being updated
}
DebouncedEvent::NoticeRemove(_) => {
// do nothing, this is sent when a file is being removed
}
DebouncedEvent::Create(p) => {
match crate::FILECACHE.add(p.as_path()) {
Ok(hash) => log::info!("[{}] Found new file @ {} ({})",thread_name, &p.as_path().to_string_lossy(), hash.to_string()),
Err(e) => log::error!("[{}] Found a new file but there was an error adding to internal cache\nFile -> {} \nError -> {:#?}", thread_name, &p.as_path().to_string_lossy(), &e),
}
}
DebouncedEvent::Write(p) => {
match crate::FILECACHE.add(p.as_path()) {
Ok(hash) => log::info!("[{}] A file was updated @ {} ({})",thread_name, &p.as_path().to_string_lossy(), hash.to_string()),
Err(e) => log::error!("[{}] Found a new file but there was an error updating the internal cache\nFile -> {} \nError -> {:#?}", thread_name, &p.as_path().to_string_lossy(), &e),
}
}
DebouncedEvent::Chmod(_) => {},
DebouncedEvent::Remove(_) => {}
DebouncedEvent::Rename(_, _) => {}
DebouncedEvent::Rescan => {},
DebouncedEvent::Error(_, _) => log::debug!(
"received a Error event on watched dir, but we don't do anything!"
),
}
}
Err(e) => log::info!("watch error: {:?}", e),
}
}
}
}

13
src/html/upload.html Executable file
View File

@@ -0,0 +1,13 @@
<!DOCTYPE html>
<html>
<body>
<h1>rfile File upload</h1>
<form action="/" method="POST" enctype=multipart/form-data target="_self">
<input type="file" id="myFile" name="data">
<input type="submit" value="Submit">
</form>
</body>
</html>

139
src/main.rs Executable file
View File

@@ -0,0 +1,139 @@
#![feature(proc_macro_hygiene, decl_macro)]
#![feature(seek_stream_len)]
#![feature(let_else)]
#[allow(unused_imports)]
#[allow(unused_macros)]
/* Logging crates */
#[macro_use]
extern crate log;
/* Parallelism and multithreading crates */
extern crate futures;
extern crate rayon;
extern crate tokio;
/* random number generation */
extern crate rand;
/* http support crates */
#[macro_use]
extern crate rocket;
extern crate mime;
/* compression crates */
//extern crate flate2;
//extern crate tar;
/* crypt crates */
extern crate data_encoding;
//extern crate ring;
/* database crates */
//extern crate rusqlite;
/* filesystem utility crates */
extern crate notify;
/* general utility crates */
//extern crate chrono;
extern crate lazy_static;
extern crate regex;
mod filecache;
mod routes;
use std::{fs, path::PathBuf};
use filecache::*;
use lazy_static::lazy_static;
use rocket::fairing::AdHoc;
static CACHE_DIR: &str = "CACHE/";
lazy_static! {
static ref FILECACHE: FileCache = {
let mut cwd = std::env::current_dir().unwrap();
cwd.push(CACHE_DIR);
FileCache::new(&cwd)
};
}
lazy_static! {
static ref CACHE_PATH: PathBuf = {
let mut cwd = std::env::current_dir().unwrap();
cwd.push(&CACHE_DIR);
cwd
};
}
#[rocket::main]
async fn main() -> Result<(), rocket::Error> {
let _rocket = rocket::build()
.mount(
"/",
rocket::routes![
routes::index,
routes::download_file,
routes::query_file,
routes::upload_file
],
)
.attach(AdHoc::on_liftoff("Application setup", |_| {
Box::pin(async move {
lazy_static::initialize(&CACHE_PATH);
lazy_static::initialize(&FILECACHE);
let keep_file = {
let mut marker_path = PathBuf::from(&CACHE_PATH.as_path());
marker_path.push("rfile.meta");
marker_path
};
if !(&CACHE_PATH.is_dir()) {
match fs::create_dir(&CACHE_PATH.as_path()) {
Ok(_) => log::info!(
"crated new cache directory @ {}",
&CACHE_PATH.as_path().to_string_lossy()
),
Err(e) => {
log::error!("Error creating cache directory - {}", &e);
panic!("{}", &e);
}
}
}
if !keep_file.exists() {
match fs::File::create(&keep_file) {
Ok(_) => log::info!(
"crated new enviroment file @ {}",
&CACHE_PATH.as_path().to_string_lossy()
),
Err(e) => {
log::error!("Error creating enviroment file - {}", &e);
panic!("{}", &e);
}
}
}
match FILECACHE.add(&keep_file) {
Ok(_) => {
log::info!("added {} to cache", &keep_file.as_path().to_string_lossy())
}
Err(e) => match e {
CacheEntryError::NotFound => todo!(),
CacheEntryError::FileLocked => todo!(),
CacheEntryError::FileExists => todo!(),
},
}
})
}))
.ignite().await?
.launch().await?;
Ok(())
}

161
src/routes/mod.rs Executable file
View File

@@ -0,0 +1,161 @@
use std::path::PathBuf;
use rocket::{Response, Request, response};
use rocket::form::Form;
use rocket::fs::{NamedFile, TempFile};
use rocket::http::{ContentType, Status};
use rocket::response::{content, Debug};
use rocket::response::Responder;
static UPLOAD_HTML: &str = include_str!("../html/upload.html");
#[get("/")]
pub async fn index() -> Result<content::RawHtml<&'static str>, Debug<std::io::Error>> {
let response = content::RawHtml(UPLOAD_HTML);
Ok(response)
}
// TODO: This should probably be moved to FileCache and a responder made for FileEntries
// currently we get a ref to a RwLock<FileEntry> that we lock for read access and grab the path
// to pass to a new NamedFile which is then wrapped in a CachedFile<T>
pub struct CachedFile(NamedFile);
// Custom responder for the wrapper CachedFile, uses the response the NamedFile would return
// and sets Content-Disposition(file type and filename) and Cache-control
impl<'r> Responder<'r, 'static> for CachedFile {
fn respond_to(self, req: &'r Request) -> response::Result<'static> {
let name = match self.0.path().file_name() {
Some(f) => {
f.to_string_lossy().to_string()
},
None => todo!()
};
Response::build_from(self.0.respond_to(req)?)
.raw_header("Content-Disposition", format!("application/octet-stream; filename=\"{}\"", name))
.raw_header("Cache-control", "max-age=86400") // 24h (24*60*60)
.ok()
}
}
#[get("/download/<file_hash>")]
pub async fn download_file(file_hash: String) -> Option<CachedFile> {
let file = match crate::FILECACHE.get(file_hash) {
Ok(fe) => fe,
Err(_) => {
panic!()
}
};
let file = {
let path_lock = file.read().unwrap().open_path();
let path = path_lock.read().unwrap().clone();
match NamedFile::open(path).await.ok() {
Some(f) => f,
None => todo!()
}
};
log::debug!("{:#?}", NamedFile::path(&file));
Some(CachedFile(file))
}
#[get("/search/<query>")]
pub async fn query_file(query: String) -> String {
// TODO: implement some sort of search function...maybe...
format!("[NOT IMPLEMENTED] Looking up {}...", &query)
}
// FIXME: Clippy suggestion (Issue is in rocket's implementation of #[derive(FromForm)] )
// warning: unnecessary closure used with `bool::then`
// --> src/routes/mod.rs:71:11
// |
// 71 | data: TempFile<'f>,
// | ^^^^^^^^^^^^ help: use `then_some(..)` instead: `then_some(data)`
// |
// = note: `#[warn(clippy::unnecessary_lazy_evaluations)]` on by default
// = help: for further information visit https://rust-lang.github.io/rust-clippy/master/index.html#unnecessary_lazy_evaluations
#[derive(FromForm)]
pub struct UploadFile<'f> {
data: TempFile<'f>,
}
#[post("/", data = "<form>")]
pub async fn upload_file(mut form: Form<UploadFile<'_>>) -> Result<String, String> {
//this is how a POST looks from our web form (simplified)
//Content-Disposition: form-data; name="filename"; filename="1510287646273.png"
//Content-Type: image/png
// <left blank>
//---DATA---
let filename = match form.data.name() {
Some(s) => String::from(s),
None => return Err(Status::BadRequest.to_string()),
};
let raw_filename = match form.data.raw_name() {
Some(s) => <&rocket::fs::FileName>::clone(&s),
None => return Err(Status::BadRequest.to_string()),
};
let filetype = match form.data.content_type() {
Some(ct) => ct.clone(),
None => ContentType::Plain,
};
let extension = match filetype.extension() {
Some(ext) => ext.to_string(),
None => {
let mut name = String::new();
let unsafe_name = raw_filename.dangerous_unsafe_unsanitized_raw();
if unsafe_name.len() < 255 {
name = match unsafe_name.as_str().trim().split_once('.') {
Some((_, ext)) => {
let mut ext = ext.replace("..", "").replace('/', "").replace(':', "");
if ext.starts_with('.') {
ext = ext[1..].to_string();
}
ext
}
None => "badext".to_string(),
};
}
log::debug!(
"unhandled extension uploaded...parse attempt ended up with {}",
&name
);
name
}
};
let filepath = PathBuf::from(format!(
"{}{}.{}",
&crate::CACHE_PATH.as_path().display(),
&filename,
extension
));
let copy_res = match form.data.copy_to(filepath.as_path()).await {
Ok(_) => Ok(()),
Err(e) => Err(e.to_string()),
};
let hasher = crate::FileEntry::new(filepath.as_path()).get_hash_string();
match (copy_res, hasher) {
(Ok(_), h) => Ok(format!(
"OK...received {}.{} size:{} bytes\nBLAKE3 Hash: {}",
&filename,
&extension,
form.data.len(),
&h
)),
(Err(_), _) => Err(Status::InternalServerError.to_string()),
}
}