Adding file metadata

This commit is contained in:
Dennis Schwerdel 2017-03-15 12:32:44 +01:00
parent d6b8e00718
commit 7e66d806b5
5 changed files with 207 additions and 6 deletions

View File

@ -19,7 +19,7 @@ mod repository;
mod algotest; mod algotest;
use chunker::ChunkerType; use chunker::ChunkerType;
use repository::{Repository, Config, Mode}; use repository::{Repository, Config, Mode, Inode};
use util::{ChecksumType, Compression, HashMethod, to_file_size}; use util::{ChecksumType, Compression, HashMethod, to_file_size};
use std::fs::File; use std::fs::File;
@ -37,6 +37,8 @@ Usage:
zvault check [--full] <repo> zvault check [--full] <repo>
zvault algotest <path> zvault algotest <path>
zvault test <repo> <path> zvault test <repo> <path>
zvault stat <path>
zvault put <repo> <path>
Options: Options:
--full Whether to verify the repository by loading all bundles --full Whether to verify the repository by loading all bundles
@ -53,8 +55,10 @@ struct Args {
cmd_info: bool, cmd_info: bool,
cmd_algotest: bool, cmd_algotest: bool,
cmd_test: bool, cmd_test: bool,
cmd_stat: bool,
cmd_check: bool, cmd_check: bool,
cmd_bundles: bool, cmd_bundles: bool,
cmd_put: bool,
arg_repo: Option<String>, arg_repo: Option<String>,
arg_path: Option<String>, arg_path: Option<String>,
flag_full: bool, flag_full: bool,
@ -91,6 +95,11 @@ fn main() {
return return
} }
if args.cmd_stat {
println!("{:?}", Inode::get_from(&args.arg_path.unwrap()).unwrap());
return
}
let mut repo = Repository::open(&args.arg_repo.unwrap()).unwrap(); let mut repo = Repository::open(&args.arg_repo.unwrap()).unwrap();
if args.cmd_check { if args.cmd_check {
@ -127,6 +136,12 @@ fn main() {
return return
} }
if args.cmd_put {
let chunks = repo.put_inode(&args.arg_path.unwrap()).unwrap();
println!("done. {} chunks, total size: {}", chunks.len(), to_file_size(chunks.iter().map(|&(_,s)| s).sum::<usize>() as u64));
return
}
if args.cmd_test { if args.cmd_test {
print!("Integrity check before..."); print!("Integrity check before...");
repo.check(true).unwrap(); repo.check(true).unwrap();

View File

@ -8,6 +8,9 @@ use ::util::Hash;
use ::chunker::{IChunker, ChunkerStatus}; use ::chunker::{IChunker, ChunkerStatus};
pub type Chunk = (Hash, usize);
impl Repository { impl Repository {
pub fn get_chunk(&mut self, hash: Hash) -> Result<Option<Vec<u8>>, &'static str> { pub fn get_chunk(&mut self, hash: Hash) -> Result<Option<Vec<u8>>, &'static str> {
// Find bundle and chunk id in index // Find bundle and chunk id in index
@ -81,12 +84,12 @@ impl Repository {
} }
#[inline] #[inline]
pub fn put_data(&mut self, mode: Mode, data: &[u8]) -> Result<Vec<(Hash, usize)>, &'static str> { pub fn put_data(&mut self, mode: Mode, data: &[u8]) -> Result<Vec<Chunk>, &'static str> {
let mut input = Cursor::new(data); let mut input = Cursor::new(data);
self.put_stream(mode, &mut input) self.put_stream(mode, &mut input)
} }
pub fn put_stream<R: Read>(&mut self, mode: Mode, data: &mut R) -> Result<Vec<(Hash, usize)>, &'static str> { pub fn put_stream<R: Read>(&mut self, mode: Mode, data: &mut R) -> Result<Vec<Chunk>, &'static str> {
let avg_size = self.config.chunker.avg_size(); let avg_size = self.config.chunker.avg_size();
let mut chunks = Vec::new(); let mut chunks = Vec::new();
let mut chunk = Vec::with_capacity(avg_size * 2); let mut chunk = Vec::with_capacity(avg_size * 2);
@ -106,14 +109,14 @@ impl Repository {
} }
#[inline] #[inline]
pub fn get_data(&mut self, chunks: &[(Hash, usize)]) -> Result<Vec<u8>, &'static str> { pub fn get_data(&mut self, chunks: &[Chunk]) -> Result<Vec<u8>, &'static str> {
let mut data = Vec::with_capacity(chunks.iter().map(|&(_, size)| size).sum()); let mut data = Vec::with_capacity(chunks.iter().map(|&(_, size)| size).sum());
try!(self.get_stream(chunks, &mut data)); try!(self.get_stream(chunks, &mut data));
Ok(data) Ok(data)
} }
#[inline] #[inline]
pub fn get_stream<W: Write>(&mut self, chunks: &[(Hash, usize)], w: &mut W) -> Result<(), &'static str> { pub fn get_stream<W: Write>(&mut self, chunks: &[Chunk], w: &mut W) -> Result<(), &'static str> {
for &(ref hash, len) in chunks { for &(ref hash, len) in chunks {
let data = try!(try!(self.get_chunk(*hash).map_err(|_| "Failed to load chunk")).ok_or("Chunk missing")); let data = try!(try!(self.get_chunk(*hash).map_err(|_| "Failed to load chunk")).ok_or("Chunk missing"));
debug_assert_eq!(data.len(), len); debug_assert_eq!(data.len(), len);

172
src/repository/metadata.rs Normal file
View File

@ -0,0 +1,172 @@
use serde::bytes::ByteBuf;
use serde::{Serialize, Deserialize};
use rmp_serde;
use std::collections::HashMap;
use std::path::Path;
use std::fs::{self, Metadata, File};
use std::os::linux::fs::MetadataExt;
use std::io::{Cursor, Read};
use ::util::Hash;
use super::{Repository, Mode, Chunk};
#[derive(Debug, Eq, PartialEq)]
pub enum FileType {
File,
Directory,
Symlink
}
serde_impl!(FileType(u8) {
File => 0,
Directory => 1,
Symlink => 2
});
#[derive(Debug)]
pub enum FileContents {
Inline(ByteBuf),
Chunked(Vec<Chunk>)
}
serde_impl!(FileContents(u8) {
Inline(ByteBuf) => 0,
Chunked(Vec<Chunk>) => 1
});
#[derive(Debug)]
pub struct Inode {
pub name: String,
pub size: u64,
pub file_type: FileType,
pub mode: u32,
pub user: u32,
pub group: u32,
pub access_time: i64,
pub modify_time: i64,
pub create_time: i64,
pub symlink_target: Option<String>,
pub contents: Option<FileContents>,
pub children: Option<HashMap<String, Vec<Chunk>>>
}
impl Default for Inode {
fn default() -> Self {
Inode {
name: "".to_string(),
size: 0,
file_type: FileType::File,
mode: 0o644,
user: 1000,
group: 1000,
access_time: 0,
modify_time: 0,
create_time: 0,
symlink_target: None,
contents: None,
children: None
}
}
}
serde_impl!(Inode(u8) {
name: String => 0,
size: u64 => 1,
file_type: FileType => 2,
mode: u32 => 3,
user: u32 => 4,
group: u32 => 5,
access_time: i64 => 6,
modify_time: i64 => 7,
create_time: i64 => 8,
symlink_target: Option<String> => 9,
contents: Option<FileContents> => 10,
children: HashMap<String, Vec<Chunk>> => 11
});
impl Inode {
fn get_extended_attrs_from(&mut self, meta: &Metadata) -> Result<(), &'static str> {
self.mode = meta.st_mode();
self.user = meta.st_uid();
self.group = meta.st_gid();
self.access_time = meta.st_atime();
self.modify_time = meta.st_mtime();
self.create_time = meta.st_ctime();
Ok(())
}
pub fn get_from<P: AsRef<Path>>(path: P) -> Result<Self, &'static str> {
let name = try!(path.as_ref().file_name().ok_or("Not a file")).to_string_lossy().to_string();
let meta = try!(fs::symlink_metadata(path.as_ref()).map_err(|_| "Failed to get metadata"));
let mut inode = Inode::default();
inode.name = name;
inode.size = meta.len();
inode.file_type = if meta.is_file() {
FileType::File
} else if meta.is_dir() {
FileType::Directory
} else if meta.file_type().is_symlink() {
FileType::Symlink
} else {
return Err("Unsupported file type");
};
if meta.file_type().is_symlink() {
inode.symlink_target = Some(try!(fs::read_link(path).map_err(|_| "Failed to read symlink")).to_string_lossy().to_string());
}
try!(inode.get_extended_attrs_from(&meta));
Ok(inode)
}
#[allow(dead_code)]
pub fn create_at<P: AsRef<Path>>(&self, path: P) -> Result<(), &'static str> {
let full_path = path.as_ref().join(&self.name);
match self.file_type {
FileType::File => {
try!(File::create(&full_path).map_err(|_| "Failed to create file"));
},
FileType::Directory => {
try!(fs::create_dir(&full_path).map_err(|_| "Failed to create directory"));
},
FileType::Symlink => {
if let Some(ref src) = self.symlink_target {
try!(fs::soft_link(src, &full_path).map_err(|_| "Failed to create symlink"));
} else {
return Err("Symlink without destination")
}
}
}
//FIXME: set times and permissions
Ok(())
}
}
impl Repository {
pub fn put_inode<P: AsRef<Path>>(&mut self, path: P) -> Result<Vec<Chunk>, &'static str> {
let mut inode = try!(Inode::get_from(path.as_ref()));
if inode.file_type == FileType::File && inode.size > 0 {
let mut file = try!(File::open(path).map_err(|_| "Failed to open file"));
if inode.size < 100 {
let mut data = Vec::with_capacity(inode.size as usize);
try!(file.read_to_end(&mut data).map_err(|_| "Failed to read file contents"));
inode.contents = Some(FileContents::Inline(data.into()));
} else {
let chunks = try!(self.put_stream(Mode::Content, &mut file));
inode.contents = Some(FileContents::Chunked(chunks));
}
}
let mut inode_data = Vec::new();
{
let mut writer = rmp_serde::Serializer::new(&mut inode_data);
inode.serialize(&mut writer).map_err(|_| "Failed to write inode data");
}
self.put_data(Mode::Meta, &inode_data)
}
#[inline]
pub fn get_inode(&mut self, chunks: &[Chunk]) -> Result<Inode, &'static str> {
let data = Cursor::new(try!(self.get_data(chunks)));
let mut reader = rmp_serde::Deserializer::new(data);
Inode::deserialize(&mut reader).map_err(|_| "Failed to read inode data")
}
}

View File

@ -3,6 +3,7 @@ mod bundle_map;
mod integrity; mod integrity;
mod basic_io; mod basic_io;
mod info; mod info;
mod metadata;
use std::mem; use std::mem;
use std::cmp::max; use std::cmp::max;
@ -14,6 +15,8 @@ use super::bundle::{BundleDb, BundleWriter};
use super::chunker::Chunker; use super::chunker::Chunker;
pub use self::config::Config; pub use self::config::Config;
pub use self::metadata::Inode;
pub use self::basic_io::Chunk;
use self::bundle_map::BundleMap; use self::bundle_map::BundleMap;

View File

@ -12,7 +12,7 @@ use std::u64;
#[repr(packed)] #[repr(packed)]
#[derive(Clone, Copy, PartialEq, Debug, Hash, Eq)] #[derive(Clone, Copy, PartialEq, Hash, Eq)]
pub struct Hash { pub struct Hash {
pub high: u64, pub high: u64,
pub low: u64 pub low: u64
@ -37,6 +37,14 @@ impl fmt::Display for Hash {
} }
} }
impl fmt::Debug for Hash {
#[inline]
fn fmt(&self, fmt: &mut fmt::Formatter) -> Result<(), fmt::Error> {
write!(fmt, "{:16x}{:16x}", self.high, self.low)
}
}
impl Serialize for Hash { impl Serialize for Hash {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error> where S: serde::Serializer { fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error> where S: serde::Serializer {
let hash = Hash{high: u64::to_le(self.high), low: u64::to_le(self.low)}; let hash = Hash{high: u64::to_le(self.high), low: u64::to_le(self.low)};